2024-11-13 01:30:47,562 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 01:30:47,580 main DEBUG Took 0.010617 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 01:30:47,581 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 01:30:47,581 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 01:30:47,582 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 01:30:47,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,591 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 01:30:47,606 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,607 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,607 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,609 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,610 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,611 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,612 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,613 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,613 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,614 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,614 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,615 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,615 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,616 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,616 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,616 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,617 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,617 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,618 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,618 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,619 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 01:30:47,619 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,620 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 01:30:47,622 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 01:30:47,623 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 01:30:47,625 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 01:30:47,626 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 01:30:47,628 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 01:30:47,628 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 01:30:47,636 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 01:30:47,639 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 01:30:47,641 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 01:30:47,642 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 01:30:47,642 main DEBUG createAppenders(={Console}) 2024-11-13 01:30:47,643 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-13 01:30:47,643 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-13 01:30:47,644 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-13 01:30:47,645 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 01:30:47,645 main DEBUG OutputStream closed 2024-11-13 01:30:47,645 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 01:30:47,646 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 01:30:47,646 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-13 01:30:47,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 01:30:47,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 01:30:47,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 01:30:47,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 01:30:47,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 01:30:47,725 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 01:30:47,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 01:30:47,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 01:30:47,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 01:30:47,726 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 01:30:47,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 01:30:47,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 01:30:47,727 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 01:30:47,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 01:30:47,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 01:30:47,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 01:30:47,728 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 01:30:47,729 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 01:30:47,731 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 01:30:47,732 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-13 01:30:47,732 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 01:30:47,733 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-13T01:30:47,744 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-13 01:30:47,747 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 01:30:47,747 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T01:30:47,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=749, ProcessCount=11, AvailableMemoryMB=2841 2024-11-13T01:30:48,039 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5 2024-11-13T01:30:48,570 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T01:30:48,658 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T01:30:48,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-13T01:30:48,761 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461448751 2024-11-13T01:30:48,831 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461448751 2024-11-13T01:30:48,903 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:48,910 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-13T01:30:48,910 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-13T01:30:48,949 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f1f5a002e11b3b012bf1f3a9bd144cd0, NAME => 'testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5 2024-11-13T01:30:48,992 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T01:30:48,996 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T01:30:49,078 INFO [StoreOpener-f1f5a002e11b3b012bf1f3a9bd144cd0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,114 INFO [StoreOpener-f1f5a002e11b3b012bf1f3a9bd144cd0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f1f5a002e11b3b012bf1f3a9bd144cd0 columnFamilyName MyCF 2024-11-13T01:30:49,122 DEBUG [StoreOpener-f1f5a002e11b3b012bf1f3a9bd144cd0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T01:30:49,129 INFO [StoreOpener-f1f5a002e11b3b012bf1f3a9bd144cd0-1 {}] regionserver.HStore(327): Store=f1f5a002e11b3b012bf1f3a9bd144cd0/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T01:30:49,132 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,137 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/data/default/testLockupAroundBadAssignSync/f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,138 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/data/default/testLockupAroundBadAssignSync/f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,140 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,141 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,146 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f1f5a002e11b3b012bf1f3a9bd144cd0 2024-11-13T01:30:49,157 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/data/default/testLockupAroundBadAssignSync/f1f5a002e11b3b012bf1f3a9bd144cd0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T01:30:49,159 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f1f5a002e11b3b012bf1f3a9bd144cd0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62379550, jitterRate=-0.07047227025032043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T01:30:49,168 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T01:30:49,186 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f1f5a002e11b3b012bf1f3a9bd144cd0: Writing region info on filesystem at 1731461449041Initializing all the Stores at 1731461449043 (+2 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1731461449044 (+1 ms)Cleaning up temporary data from old regions at 1731461449141 (+97 ms)Region opened successfully at 1731461449183 (+42 ms) 2024-11-13T01:30:49,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461448751) roll requested 2024-11-13T01:30:49,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449240 2024-11-13T01:30:49,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461448751 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449240 2024-11-13T01:30:49,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449240) roll requested 2024-11-13T01:30:49,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449253 2024-11-13T01:30:49,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449253 2024-11-13T01:30:49,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449253) roll requested 2024-11-13T01:30:49,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449265 2024-11-13T01:30:49,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449265 2024-11-13T01:30:49,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449265) roll requested 2024-11-13T01:30:49,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449276 2024-11-13T01:30:49,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449276 2024-11-13T01:30:49,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449276) roll requested 2024-11-13T01:30:49,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449285 2024-11-13T01:30:49,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,293 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449285 2024-11-13T01:30:49,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449285) roll requested 2024-11-13T01:30:49,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449298 2024-11-13T01:30:49,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449298 2024-11-13T01:30:49,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449298) roll requested 2024-11-13T01:30:49,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449310 2024-11-13T01:30:49,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449310 2024-11-13T01:30:49,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449310) roll requested 2024-11-13T01:30:49,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449320 2024-11-13T01:30:49,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449320 2024-11-13T01:30:49,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449320) roll requested 2024-11-13T01:30:49,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449330 2024-11-13T01:30:49,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449330 2024-11-13T01:30:49,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449330) roll requested 2024-11-13T01:30:49,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449340 2024-11-13T01:30:49,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449340 2024-11-13T01:30:49,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449340) roll requested 2024-11-13T01:30:49,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449350 2024-11-13T01:30:49,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449350 2024-11-13T01:30:49,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449350) roll requested 2024-11-13T01:30:49,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449361 2024-11-13T01:30:49,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449361 2024-11-13T01:30:49,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449361) roll requested 2024-11-13T01:30:49,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449370 2024-11-13T01:30:49,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449370 2024-11-13T01:30:49,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449370) roll requested 2024-11-13T01:30:49,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449381 2024-11-13T01:30:49,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449381 2024-11-13T01:30:49,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449381) roll requested 2024-11-13T01:30:49,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449393 2024-11-13T01:30:49,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449393 2024-11-13T01:30:49,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449393) roll requested 2024-11-13T01:30:49,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449404 2024-11-13T01:30:49,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449404 2024-11-13T01:30:49,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449404) roll requested 2024-11-13T01:30:49,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449415 2024-11-13T01:30:49,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449415 2024-11-13T01:30:49,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449415) roll requested 2024-11-13T01:30:49,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449425 2024-11-13T01:30:49,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449425 2024-11-13T01:30:49,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449425) roll requested 2024-11-13T01:30:49,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449437 2024-11-13T01:30:49,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449437 2024-11-13T01:30:49,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449437) roll requested 2024-11-13T01:30:49,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449447 2024-11-13T01:30:49,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449447 2024-11-13T01:30:49,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449447) roll requested 2024-11-13T01:30:49,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449457 2024-11-13T01:30:49,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449457 2024-11-13T01:30:49,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449457) roll requested 2024-11-13T01:30:49,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449470 2024-11-13T01:30:49,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449470 2024-11-13T01:30:49,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449470) roll requested 2024-11-13T01:30:49,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449482 2024-11-13T01:30:49,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449482 2024-11-13T01:30:49,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449482) roll requested 2024-11-13T01:30:49,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449496 2024-11-13T01:30:49,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449496 2024-11-13T01:30:49,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449496) roll requested 2024-11-13T01:30:49,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449505 2024-11-13T01:30:49,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449505 2024-11-13T01:30:49,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449505) roll requested 2024-11-13T01:30:49,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449512 2024-11-13T01:30:49,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449512 2024-11-13T01:30:49,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449512) roll requested 2024-11-13T01:30:49,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449521 2024-11-13T01:30:49,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449521 2024-11-13T01:30:49,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449521) roll requested 2024-11-13T01:30:49,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449530 2024-11-13T01:30:49,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449530 2024-11-13T01:30:49,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449530) roll requested 2024-11-13T01:30:49,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449542 2024-11-13T01:30:49,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449542 2024-11-13T01:30:49,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449542) roll requested 2024-11-13T01:30:49,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449555 2024-11-13T01:30:49,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449555 2024-11-13T01:30:49,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449555) roll requested 2024-11-13T01:30:49,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449563 2024-11-13T01:30:49,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449563 2024-11-13T01:30:49,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449563) roll requested 2024-11-13T01:30:49,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449571 2024-11-13T01:30:49,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449571 2024-11-13T01:30:49,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449571) roll requested 2024-11-13T01:30:49,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449582 2024-11-13T01:30:49,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449582 2024-11-13T01:30:49,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449582) roll requested 2024-11-13T01:30:49,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449592 2024-11-13T01:30:49,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449592 2024-11-13T01:30:49,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449592) roll requested 2024-11-13T01:30:49,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449600 2024-11-13T01:30:49,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449600 2024-11-13T01:30:49,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449600) roll requested 2024-11-13T01:30:49,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449609 2024-11-13T01:30:49,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449600 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449609 2024-11-13T01:30:49,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449609) roll requested 2024-11-13T01:30:49,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449618 2024-11-13T01:30:49,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449618 2024-11-13T01:30:49,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449618) roll requested 2024-11-13T01:30:49,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449631 2024-11-13T01:30:49,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449631 2024-11-13T01:30:49,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449631) roll requested 2024-11-13T01:30:49,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449639 2024-11-13T01:30:49,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449639 2024-11-13T01:30:49,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449639) roll requested 2024-11-13T01:30:49,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449647 2024-11-13T01:30:49,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449647 2024-11-13T01:30:49,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449647) roll requested 2024-11-13T01:30:49,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449655 2024-11-13T01:30:49,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449647 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449655 2024-11-13T01:30:49,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449655) roll requested 2024-11-13T01:30:49,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449662 2024-11-13T01:30:49,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449662 2024-11-13T01:30:49,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449662) roll requested 2024-11-13T01:30:49,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449671 2024-11-13T01:30:49,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449671 2024-11-13T01:30:49,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449671) roll requested 2024-11-13T01:30:49,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449681 2024-11-13T01:30:49,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449681 2024-11-13T01:30:49,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449681) roll requested 2024-11-13T01:30:49,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449690 2024-11-13T01:30:49,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449690 2024-11-13T01:30:49,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449690) roll requested 2024-11-13T01:30:49,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449698 2024-11-13T01:30:49,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449698 2024-11-13T01:30:49,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449698) roll requested 2024-11-13T01:30:49,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449705 2024-11-13T01:30:49,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449705 2024-11-13T01:30:49,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449705) roll requested 2024-11-13T01:30:49,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449712 2024-11-13T01:30:49,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449712 2024-11-13T01:30:49,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449712) roll requested 2024-11-13T01:30:49,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449722 2024-11-13T01:30:49,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449722 2024-11-13T01:30:49,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449722) roll requested 2024-11-13T01:30:49,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449735 2024-11-13T01:30:49,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449735 2024-11-13T01:30:49,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449735) roll requested 2024-11-13T01:30:49,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449742 2024-11-13T01:30:49,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449742 2024-11-13T01:30:49,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449742) roll requested 2024-11-13T01:30:49,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449750 2024-11-13T01:30:49,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449750 2024-11-13T01:30:49,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,758 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449750) roll requested 2024-11-13T01:30:49,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449759 2024-11-13T01:30:49,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449759 2024-11-13T01:30:49,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449759) roll requested 2024-11-13T01:30:49,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449767 2024-11-13T01:30:49,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449767 2024-11-13T01:30:49,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449767) roll requested 2024-11-13T01:30:49,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449777 2024-11-13T01:30:49,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449777 2024-11-13T01:30:49,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449777) roll requested 2024-11-13T01:30:49,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449791 2024-11-13T01:30:49,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449791 2024-11-13T01:30:49,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449791) roll requested 2024-11-13T01:30:49,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449799 2024-11-13T01:30:49,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449799 2024-11-13T01:30:49,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449799) roll requested 2024-11-13T01:30:49,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449809 2024-11-13T01:30:49,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449809 2024-11-13T01:30:49,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449809) roll requested 2024-11-13T01:30:49,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449816 2024-11-13T01:30:49,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449816 2024-11-13T01:30:49,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449816) roll requested 2024-11-13T01:30:49,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449824 2024-11-13T01:30:49,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449824 2024-11-13T01:30:49,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449824) roll requested 2024-11-13T01:30:49,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449832 2024-11-13T01:30:49,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449832 2024-11-13T01:30:49,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449832) roll requested 2024-11-13T01:30:49,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449841 2024-11-13T01:30:49,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449841 2024-11-13T01:30:49,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449841) roll requested 2024-11-13T01:30:49,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449851 2024-11-13T01:30:49,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449851 2024-11-13T01:30:49,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449851) roll requested 2024-11-13T01:30:49,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449862 2024-11-13T01:30:49,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449862 2024-11-13T01:30:49,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449862) roll requested 2024-11-13T01:30:49,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449878 2024-11-13T01:30:49,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449878 2024-11-13T01:30:49,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449878) roll requested 2024-11-13T01:30:49,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449891 2024-11-13T01:30:49,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449891 2024-11-13T01:30:49,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449891) roll requested 2024-11-13T01:30:49,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449898 2024-11-13T01:30:49,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449898 2024-11-13T01:30:49,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449898) roll requested 2024-11-13T01:30:49,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449905 2024-11-13T01:30:49,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449905 2024-11-13T01:30:49,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449905) roll requested 2024-11-13T01:30:49,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449912 2024-11-13T01:30:49,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449905 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449912 2024-11-13T01:30:49,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449912) roll requested 2024-11-13T01:30:49,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449920 2024-11-13T01:30:49,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449920 2024-11-13T01:30:49,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449920) roll requested 2024-11-13T01:30:49,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449927 2024-11-13T01:30:49,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449927 2024-11-13T01:30:49,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449927) roll requested 2024-11-13T01:30:49,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449936 2024-11-13T01:30:49,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449936 2024-11-13T01:30:49,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449936) roll requested 2024-11-13T01:30:49,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449946 2024-11-13T01:30:49,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449946 2024-11-13T01:30:49,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449946) roll requested 2024-11-13T01:30:49,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449956 2024-11-13T01:30:49,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449946 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449956 2024-11-13T01:30:49,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449956) roll requested 2024-11-13T01:30:49,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449966 2024-11-13T01:30:49,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449966 2024-11-13T01:30:49,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449966) roll requested 2024-11-13T01:30:49,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449975 2024-11-13T01:30:49,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449975 2024-11-13T01:30:49,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449975) roll requested 2024-11-13T01:30:49,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449984 2024-11-13T01:30:49,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:49,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449984 2024-11-13T01:30:49,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:49,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:49,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449984) roll requested 2024-11-13T01:30:49,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461449993 2024-11-13T01:30:50,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449993 2024-11-13T01:30:50,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461449993) roll requested 2024-11-13T01:30:50,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450002 2024-11-13T01:30:50,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461449993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450002 2024-11-13T01:30:50,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450002) roll requested 2024-11-13T01:30:50,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450009 2024-11-13T01:30:50,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450009 2024-11-13T01:30:50,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450009) roll requested 2024-11-13T01:30:50,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450019 2024-11-13T01:30:50,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450019 2024-11-13T01:30:50,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450019) roll requested 2024-11-13T01:30:50,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450029 2024-11-13T01:30:50,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450029 2024-11-13T01:30:50,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450029) roll requested 2024-11-13T01:30:50,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450038 2024-11-13T01:30:50,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450038 2024-11-13T01:30:50,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450038) roll requested 2024-11-13T01:30:50,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450050 2024-11-13T01:30:50,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450050 2024-11-13T01:30:50,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450050) roll requested 2024-11-13T01:30:50,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450059 2024-11-13T01:30:50,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450059 2024-11-13T01:30:50,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,067 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450059) roll requested 2024-11-13T01:30:50,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450067 2024-11-13T01:30:50,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450067 2024-11-13T01:30:50,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450067) roll requested 2024-11-13T01:30:50,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450077 2024-11-13T01:30:50,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450077 2024-11-13T01:30:50,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450077) roll requested 2024-11-13T01:30:50,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450086 2024-11-13T01:30:50,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450086 2024-11-13T01:30:50,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450086) roll requested 2024-11-13T01:30:50,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450099 2024-11-13T01:30:50,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450099 2024-11-13T01:30:50,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450099) roll requested 2024-11-13T01:30:50,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450109 2024-11-13T01:30:50,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450109 2024-11-13T01:30:50,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450109) roll requested 2024-11-13T01:30:50,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450120 2024-11-13T01:30:50,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450120 2024-11-13T01:30:50,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450120) roll requested 2024-11-13T01:30:50,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450133 2024-11-13T01:30:50,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450133 2024-11-13T01:30:50,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450133) roll requested 2024-11-13T01:30:50,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450143 2024-11-13T01:30:50,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450143 2024-11-13T01:30:50,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450143) roll requested 2024-11-13T01:30:50,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450153 2024-11-13T01:30:50,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450153 2024-11-13T01:30:50,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450153) roll requested 2024-11-13T01:30:50,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450162 2024-11-13T01:30:50,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450162 2024-11-13T01:30:50,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450162) roll requested 2024-11-13T01:30:50,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450172 2024-11-13T01:30:50,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450172 2024-11-13T01:30:50,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450172) roll requested 2024-11-13T01:30:50,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450180 2024-11-13T01:30:50,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450180 2024-11-13T01:30:50,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450180) roll requested 2024-11-13T01:30:50,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450187 2024-11-13T01:30:50,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450187 2024-11-13T01:30:50,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450187) roll requested 2024-11-13T01:30:50,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450195 2024-11-13T01:30:50,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450195 2024-11-13T01:30:50,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450195) roll requested 2024-11-13T01:30:50,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450204 2024-11-13T01:30:50,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450204 2024-11-13T01:30:50,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450204) roll requested 2024-11-13T01:30:50,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450213 2024-11-13T01:30:50,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450213 2024-11-13T01:30:50,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450213) roll requested 2024-11-13T01:30:50,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450221 2024-11-13T01:30:50,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450221 2024-11-13T01:30:50,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450221) roll requested 2024-11-13T01:30:50,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450229 2024-11-13T01:30:50,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450229 2024-11-13T01:30:50,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450229) roll requested 2024-11-13T01:30:50,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450237 2024-11-13T01:30:50,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450237 2024-11-13T01:30:50,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450237) roll requested 2024-11-13T01:30:50,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450244 2024-11-13T01:30:50,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450244 2024-11-13T01:30:50,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450244) roll requested 2024-11-13T01:30:50,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450255 2024-11-13T01:30:50,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450255 2024-11-13T01:30:50,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450255) roll requested 2024-11-13T01:30:50,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450266 2024-11-13T01:30:50,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450266 2024-11-13T01:30:50,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450266) roll requested 2024-11-13T01:30:50,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450277 2024-11-13T01:30:50,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450277 2024-11-13T01:30:50,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450277) roll requested 2024-11-13T01:30:50,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450289 2024-11-13T01:30:50,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450289 2024-11-13T01:30:50,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450289) roll requested 2024-11-13T01:30:50,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450301 2024-11-13T01:30:50,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450301 2024-11-13T01:30:50,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450301) roll requested 2024-11-13T01:30:50,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450312 2024-11-13T01:30:50,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450312 2024-11-13T01:30:50,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450312) roll requested 2024-11-13T01:30:50,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450318 2024-11-13T01:30:50,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450318 2024-11-13T01:30:50,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450318) roll requested 2024-11-13T01:30:50,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450329 2024-11-13T01:30:50,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450329 2024-11-13T01:30:50,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450329) roll requested 2024-11-13T01:30:50,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450337 2024-11-13T01:30:50,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450337 2024-11-13T01:30:50,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450337) roll requested 2024-11-13T01:30:50,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450346 2024-11-13T01:30:50,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450346 2024-11-13T01:30:50,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450346) roll requested 2024-11-13T01:30:50,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450355 2024-11-13T01:30:50,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450355 2024-11-13T01:30:50,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450355) roll requested 2024-11-13T01:30:50,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450365 2024-11-13T01:30:50,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450365 2024-11-13T01:30:50,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450365) roll requested 2024-11-13T01:30:50,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450374 2024-11-13T01:30:50,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450374 2024-11-13T01:30:50,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450374) roll requested 2024-11-13T01:30:50,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450385 2024-11-13T01:30:50,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450385 2024-11-13T01:30:50,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450385) roll requested 2024-11-13T01:30:50,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450394 2024-11-13T01:30:50,400 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450394 2024-11-13T01:30:50,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450394) roll requested 2024-11-13T01:30:50,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450401 2024-11-13T01:30:50,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450401 2024-11-13T01:30:50,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450401) roll requested 2024-11-13T01:30:50,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450409 2024-11-13T01:30:50,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450409 2024-11-13T01:30:50,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450409) roll requested 2024-11-13T01:30:50,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450417 2024-11-13T01:30:50,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450417 2024-11-13T01:30:50,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450417) roll requested 2024-11-13T01:30:50,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450427 2024-11-13T01:30:50,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450427 2024-11-13T01:30:50,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450427) roll requested 2024-11-13T01:30:50,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450438 2024-11-13T01:30:50,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450438 2024-11-13T01:30:50,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450438) roll requested 2024-11-13T01:30:50,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450449 2024-11-13T01:30:50,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450449 2024-11-13T01:30:50,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450449) roll requested 2024-11-13T01:30:50,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450456 2024-11-13T01:30:50,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450456 2024-11-13T01:30:50,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450456) roll requested 2024-11-13T01:30:50,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450463 2024-11-13T01:30:50,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450463 2024-11-13T01:30:50,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450463) roll requested 2024-11-13T01:30:50,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450470 2024-11-13T01:30:50,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450470 2024-11-13T01:30:50,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450470) roll requested 2024-11-13T01:30:50,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450480 2024-11-13T01:30:50,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450480 2024-11-13T01:30:50,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450480) roll requested 2024-11-13T01:30:50,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450488 2024-11-13T01:30:50,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450488 2024-11-13T01:30:50,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450488) roll requested 2024-11-13T01:30:50,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450496 2024-11-13T01:30:50,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450496 2024-11-13T01:30:50,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450496) roll requested 2024-11-13T01:30:50,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450505 2024-11-13T01:30:50,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450505 2024-11-13T01:30:50,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450505) roll requested 2024-11-13T01:30:50,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450512 2024-11-13T01:30:50,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450512 2024-11-13T01:30:50,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450512) roll requested 2024-11-13T01:30:50,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450521 2024-11-13T01:30:50,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450521 2024-11-13T01:30:50,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450521) roll requested 2024-11-13T01:30:50,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450529 2024-11-13T01:30:50,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450529 2024-11-13T01:30:50,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450529) roll requested 2024-11-13T01:30:50,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450537 2024-11-13T01:30:50,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450537 2024-11-13T01:30:50,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450537) roll requested 2024-11-13T01:30:50,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450544 2024-11-13T01:30:50,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450544 2024-11-13T01:30:50,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450544) roll requested 2024-11-13T01:30:50,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450551 2024-11-13T01:30:50,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450551 2024-11-13T01:30:50,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450551) roll requested 2024-11-13T01:30:50,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450558 2024-11-13T01:30:50,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450558 2024-11-13T01:30:50,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450558) roll requested 2024-11-13T01:30:50,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450565 2024-11-13T01:30:50,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450565 2024-11-13T01:30:50,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450565) roll requested 2024-11-13T01:30:50,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450571 2024-11-13T01:30:50,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450571 2024-11-13T01:30:50,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,578 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450571) roll requested 2024-11-13T01:30:50,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450579 2024-11-13T01:30:50,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450579 2024-11-13T01:30:50,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450579) roll requested 2024-11-13T01:30:50,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450587 2024-11-13T01:30:50,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450587 2024-11-13T01:30:50,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450587) roll requested 2024-11-13T01:30:50,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450597 2024-11-13T01:30:50,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450597 2024-11-13T01:30:50,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450597) roll requested 2024-11-13T01:30:50,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450604 2024-11-13T01:30:50,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450604 2024-11-13T01:30:50,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450604) roll requested 2024-11-13T01:30:50,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450611 2024-11-13T01:30:50,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450611 2024-11-13T01:30:50,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450611) roll requested 2024-11-13T01:30:50,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450617 2024-11-13T01:30:50,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450617 2024-11-13T01:30:50,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450617) roll requested 2024-11-13T01:30:50,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450623 2024-11-13T01:30:50,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450623 2024-11-13T01:30:50,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450623) roll requested 2024-11-13T01:30:50,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450632 2024-11-13T01:30:50,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450632 2024-11-13T01:30:50,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450632) roll requested 2024-11-13T01:30:50,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450641 2024-11-13T01:30:50,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450641 2024-11-13T01:30:50,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450641) roll requested 2024-11-13T01:30:50,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450649 2024-11-13T01:30:50,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450649 2024-11-13T01:30:50,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450649) roll requested 2024-11-13T01:30:50,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450656 2024-11-13T01:30:50,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450656 2024-11-13T01:30:50,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450656) roll requested 2024-11-13T01:30:50,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450667 2024-11-13T01:30:50,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450667 2024-11-13T01:30:50,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450667) roll requested 2024-11-13T01:30:50,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450676 2024-11-13T01:30:50,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450676 2024-11-13T01:30:50,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450676) roll requested 2024-11-13T01:30:50,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450683 2024-11-13T01:30:50,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450683 2024-11-13T01:30:50,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450683) roll requested 2024-11-13T01:30:50,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450691 2024-11-13T01:30:50,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450691 2024-11-13T01:30:50,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450691) roll requested 2024-11-13T01:30:50,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450699 2024-11-13T01:30:50,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450699 2024-11-13T01:30:50,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450699) roll requested 2024-11-13T01:30:50,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450706 2024-11-13T01:30:50,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450706 2024-11-13T01:30:50,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450706) roll requested 2024-11-13T01:30:50,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450713 2024-11-13T01:30:50,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450713 2024-11-13T01:30:50,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450713) roll requested 2024-11-13T01:30:50,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450725 2024-11-13T01:30:50,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450725 2024-11-13T01:30:50,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450725) roll requested 2024-11-13T01:30:50,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450736 2024-11-13T01:30:50,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450736 2024-11-13T01:30:50,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450736) roll requested 2024-11-13T01:30:50,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450745 2024-11-13T01:30:50,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450745 2024-11-13T01:30:50,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450745) roll requested 2024-11-13T01:30:50,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450754 2024-11-13T01:30:50,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450754 2024-11-13T01:30:50,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450754) roll requested 2024-11-13T01:30:50,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450763 2024-11-13T01:30:50,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450763 2024-11-13T01:30:50,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450763) roll requested 2024-11-13T01:30:50,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450774 2024-11-13T01:30:50,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450774 2024-11-13T01:30:50,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450774) roll requested 2024-11-13T01:30:50,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450782 2024-11-13T01:30:50,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450782 2024-11-13T01:30:50,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450782) roll requested 2024-11-13T01:30:50,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450790 2024-11-13T01:30:50,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450790 2024-11-13T01:30:50,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450790) roll requested 2024-11-13T01:30:50,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450798 2024-11-13T01:30:50,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450798 2024-11-13T01:30:50,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450798) roll requested 2024-11-13T01:30:50,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450809 2024-11-13T01:30:50,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450809 2024-11-13T01:30:50,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450809) roll requested 2024-11-13T01:30:50,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450821 2024-11-13T01:30:50,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450821 2024-11-13T01:30:50,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450821) roll requested 2024-11-13T01:30:50,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450828 2024-11-13T01:30:50,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450828 2024-11-13T01:30:50,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450828) roll requested 2024-11-13T01:30:50,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450835 2024-11-13T01:30:50,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450835 2024-11-13T01:30:50,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450835) roll requested 2024-11-13T01:30:50,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450842 2024-11-13T01:30:50,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450842 2024-11-13T01:30:50,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450842) roll requested 2024-11-13T01:30:50,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450850 2024-11-13T01:30:50,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450850 2024-11-13T01:30:50,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450850) roll requested 2024-11-13T01:30:50,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450858 2024-11-13T01:30:50,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450858 2024-11-13T01:30:50,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450858) roll requested 2024-11-13T01:30:50,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450869 2024-11-13T01:30:50,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450869 2024-11-13T01:30:50,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450869) roll requested 2024-11-13T01:30:50,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450876 2024-11-13T01:30:50,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450876 2024-11-13T01:30:50,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450876) roll requested 2024-11-13T01:30:50,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450885 2024-11-13T01:30:50,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450885 2024-11-13T01:30:50,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450885) roll requested 2024-11-13T01:30:50,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450893 2024-11-13T01:30:50,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450893 2024-11-13T01:30:50,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450893) roll requested 2024-11-13T01:30:50,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450900 2024-11-13T01:30:50,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450900 2024-11-13T01:30:50,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450900) roll requested 2024-11-13T01:30:50,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450907 2024-11-13T01:30:50,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450907 2024-11-13T01:30:50,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450907) roll requested 2024-11-13T01:30:50,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450916 2024-11-13T01:30:50,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450916 2024-11-13T01:30:50,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450916) roll requested 2024-11-13T01:30:50,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450925 2024-11-13T01:30:50,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450925 2024-11-13T01:30:50,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450925) roll requested 2024-11-13T01:30:50,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450937 2024-11-13T01:30:50,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450937 2024-11-13T01:30:50,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450937) roll requested 2024-11-13T01:30:50,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450943 2024-11-13T01:30:50,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450943 2024-11-13T01:30:50,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450943) roll requested 2024-11-13T01:30:50,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450950 2024-11-13T01:30:50,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450950 2024-11-13T01:30:50,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450950) roll requested 2024-11-13T01:30:50,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450956 2024-11-13T01:30:50,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450956 2024-11-13T01:30:50,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450956) roll requested 2024-11-13T01:30:50,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450961 2024-11-13T01:30:50,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450961 2024-11-13T01:30:50,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450961) roll requested 2024-11-13T01:30:50,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450968 2024-11-13T01:30:50,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450968 2024-11-13T01:30:50,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450968) roll requested 2024-11-13T01:30:50,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450977 2024-11-13T01:30:50,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450977 2024-11-13T01:30:50,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450977) roll requested 2024-11-13T01:30:50,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450985 2024-11-13T01:30:50,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:50,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450985 2024-11-13T01:30:50,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:50,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:50,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450985) roll requested 2024-11-13T01:30:50,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461450996 2024-11-13T01:30:51,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450996 2024-11-13T01:30:51,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461450996) roll requested 2024-11-13T01:30:51,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451004 2024-11-13T01:30:51,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461450996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451004 2024-11-13T01:30:51,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451004) roll requested 2024-11-13T01:30:51,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451011 2024-11-13T01:30:51,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451011 2024-11-13T01:30:51,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451011) roll requested 2024-11-13T01:30:51,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451018 2024-11-13T01:30:51,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451018 2024-11-13T01:30:51,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451018) roll requested 2024-11-13T01:30:51,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451024 2024-11-13T01:30:51,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451024 2024-11-13T01:30:51,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451024) roll requested 2024-11-13T01:30:51,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451030 2024-11-13T01:30:51,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451030 2024-11-13T01:30:51,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451030) roll requested 2024-11-13T01:30:51,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451037 2024-11-13T01:30:51,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451037 2024-11-13T01:30:51,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451037) roll requested 2024-11-13T01:30:51,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451043 2024-11-13T01:30:51,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451043 2024-11-13T01:30:51,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451043) roll requested 2024-11-13T01:30:51,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451049 2024-11-13T01:30:51,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451049 2024-11-13T01:30:51,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451049) roll requested 2024-11-13T01:30:51,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451056 2024-11-13T01:30:51,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451056 2024-11-13T01:30:51,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451056) roll requested 2024-11-13T01:30:51,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451068 2024-11-13T01:30:51,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451068 2024-11-13T01:30:51,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451068) roll requested 2024-11-13T01:30:51,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451075 2024-11-13T01:30:51,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451075 2024-11-13T01:30:51,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451075) roll requested 2024-11-13T01:30:51,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451084 2024-11-13T01:30:51,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451084 2024-11-13T01:30:51,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451084) roll requested 2024-11-13T01:30:51,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451090 2024-11-13T01:30:51,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451090 2024-11-13T01:30:51,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451090) roll requested 2024-11-13T01:30:51,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451097 2024-11-13T01:30:51,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451097 2024-11-13T01:30:51,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451097) roll requested 2024-11-13T01:30:51,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451104 2024-11-13T01:30:51,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451104 2024-11-13T01:30:51,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451104) roll requested 2024-11-13T01:30:51,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451111 2024-11-13T01:30:51,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451111 2024-11-13T01:30:51,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451111) roll requested 2024-11-13T01:30:51,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451117 2024-11-13T01:30:51,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451117 2024-11-13T01:30:51,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451117) roll requested 2024-11-13T01:30:51,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451123 2024-11-13T01:30:51,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451123 2024-11-13T01:30:51,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451123) roll requested 2024-11-13T01:30:51,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451130 2024-11-13T01:30:51,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451130 2024-11-13T01:30:51,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451130) roll requested 2024-11-13T01:30:51,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451137 2024-11-13T01:30:51,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451137 2024-11-13T01:30:51,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451137) roll requested 2024-11-13T01:30:51,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451144 2024-11-13T01:30:51,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451144 2024-11-13T01:30:51,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451144) roll requested 2024-11-13T01:30:51,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451151 2024-11-13T01:30:51,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451151 2024-11-13T01:30:51,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451151) roll requested 2024-11-13T01:30:51,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451158 2024-11-13T01:30:51,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451158 2024-11-13T01:30:51,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451158) roll requested 2024-11-13T01:30:51,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451164 2024-11-13T01:30:51,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451164 2024-11-13T01:30:51,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451164) roll requested 2024-11-13T01:30:51,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451173 2024-11-13T01:30:51,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451173 2024-11-13T01:30:51,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451173) roll requested 2024-11-13T01:30:51,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451181 2024-11-13T01:30:51,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451181 2024-11-13T01:30:51,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451181) roll requested 2024-11-13T01:30:51,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451188 2024-11-13T01:30:51,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451188 2024-11-13T01:30:51,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451188) roll requested 2024-11-13T01:30:51,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451195 2024-11-13T01:30:51,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451195 2024-11-13T01:30:51,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451195) roll requested 2024-11-13T01:30:51,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451201 2024-11-13T01:30:51,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451201 2024-11-13T01:30:51,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451201) roll requested 2024-11-13T01:30:51,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451207 2024-11-13T01:30:51,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451207 2024-11-13T01:30:51,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451207) roll requested 2024-11-13T01:30:51,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451215 2024-11-13T01:30:51,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451215 2024-11-13T01:30:51,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451215) roll requested 2024-11-13T01:30:51,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451221 2024-11-13T01:30:51,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451221 2024-11-13T01:30:51,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451221) roll requested 2024-11-13T01:30:51,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451229 2024-11-13T01:30:51,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451229 2024-11-13T01:30:51,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451229) roll requested 2024-11-13T01:30:51,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451237 2024-11-13T01:30:51,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451237 2024-11-13T01:30:51,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451237) roll requested 2024-11-13T01:30:51,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451247 2024-11-13T01:30:51,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451247 2024-11-13T01:30:51,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451247) roll requested 2024-11-13T01:30:51,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451255 2024-11-13T01:30:51,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451255 2024-11-13T01:30:51,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451255) roll requested 2024-11-13T01:30:51,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451264 2024-11-13T01:30:51,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451264 2024-11-13T01:30:51,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451264) roll requested 2024-11-13T01:30:51,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451274 2024-11-13T01:30:51,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451274 2024-11-13T01:30:51,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451274) roll requested 2024-11-13T01:30:51,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451281 2024-11-13T01:30:51,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451281 2024-11-13T01:30:51,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451281) roll requested 2024-11-13T01:30:51,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451287 2024-11-13T01:30:51,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451287 2024-11-13T01:30:51,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451287) roll requested 2024-11-13T01:30:51,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451296 2024-11-13T01:30:51,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451296 2024-11-13T01:30:51,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451296) roll requested 2024-11-13T01:30:51,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451308 2024-11-13T01:30:51,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451308 2024-11-13T01:30:51,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451308) roll requested 2024-11-13T01:30:51,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451315 2024-11-13T01:30:51,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451315 2024-11-13T01:30:51,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451315) roll requested 2024-11-13T01:30:51,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451322 2024-11-13T01:30:51,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451322 2024-11-13T01:30:51,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451322) roll requested 2024-11-13T01:30:51,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451332 2024-11-13T01:30:51,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451332 2024-11-13T01:30:51,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451332) roll requested 2024-11-13T01:30:51,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451340 2024-11-13T01:30:51,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451340 2024-11-13T01:30:51,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451340) roll requested 2024-11-13T01:30:51,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451346 2024-11-13T01:30:51,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451346 2024-11-13T01:30:51,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451346) roll requested 2024-11-13T01:30:51,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451354 2024-11-13T01:30:51,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451354 2024-11-13T01:30:51,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451354) roll requested 2024-11-13T01:30:51,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451363 2024-11-13T01:30:51,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451363 2024-11-13T01:30:51,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451363) roll requested 2024-11-13T01:30:51,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451371 2024-11-13T01:30:51,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451371 2024-11-13T01:30:51,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451371) roll requested 2024-11-13T01:30:51,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451385 2024-11-13T01:30:51,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451385 2024-11-13T01:30:51,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451385) roll requested 2024-11-13T01:30:51,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451393 2024-11-13T01:30:51,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451393 2024-11-13T01:30:51,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451393) roll requested 2024-11-13T01:30:51,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451399 2024-11-13T01:30:51,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451399 2024-11-13T01:30:51,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451399) roll requested 2024-11-13T01:30:51,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451405 2024-11-13T01:30:51,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451405 2024-11-13T01:30:51,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451405) roll requested 2024-11-13T01:30:51,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451415 2024-11-13T01:30:51,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451415 2024-11-13T01:30:51,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451415) roll requested 2024-11-13T01:30:51,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451423 2024-11-13T01:30:51,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451423 2024-11-13T01:30:51,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451423) roll requested 2024-11-13T01:30:51,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451430 2024-11-13T01:30:51,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451430 2024-11-13T01:30:51,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451430) roll requested 2024-11-13T01:30:51,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451438 2024-11-13T01:30:51,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451438 2024-11-13T01:30:51,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451438) roll requested 2024-11-13T01:30:51,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451449 2024-11-13T01:30:51,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451449 2024-11-13T01:30:51,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451449) roll requested 2024-11-13T01:30:51,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451459 2024-11-13T01:30:51,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451459 2024-11-13T01:30:51,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451459) roll requested 2024-11-13T01:30:51,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451466 2024-11-13T01:30:51,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451466 2024-11-13T01:30:51,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451466) roll requested 2024-11-13T01:30:51,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451477 2024-11-13T01:30:51,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451477 2024-11-13T01:30:51,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451477) roll requested 2024-11-13T01:30:51,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451484 2024-11-13T01:30:51,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451477 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451484 2024-11-13T01:30:51,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451484) roll requested 2024-11-13T01:30:51,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451492 2024-11-13T01:30:51,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451492 2024-11-13T01:30:51,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451492) roll requested 2024-11-13T01:30:51,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451500 2024-11-13T01:30:51,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451500 2024-11-13T01:30:51,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451500) roll requested 2024-11-13T01:30:51,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451515 2024-11-13T01:30:51,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451515 2024-11-13T01:30:51,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451515) roll requested 2024-11-13T01:30:51,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451522 2024-11-13T01:30:51,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451522 2024-11-13T01:30:51,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451522) roll requested 2024-11-13T01:30:51,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451528 2024-11-13T01:30:51,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451528 2024-11-13T01:30:51,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451528) roll requested 2024-11-13T01:30:51,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451534 2024-11-13T01:30:51,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451534 2024-11-13T01:30:51,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451534) roll requested 2024-11-13T01:30:51,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451544 2024-11-13T01:30:51,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451534 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451544 2024-11-13T01:30:51,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451544) roll requested 2024-11-13T01:30:51,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451549 2024-11-13T01:30:51,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451549 2024-11-13T01:30:51,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451549) roll requested 2024-11-13T01:30:51,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451555 2024-11-13T01:30:51,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451555 2024-11-13T01:30:51,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451555) roll requested 2024-11-13T01:30:51,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451564 2024-11-13T01:30:51,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451564 2024-11-13T01:30:51,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451564) roll requested 2024-11-13T01:30:51,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451570 2024-11-13T01:30:51,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451570 2024-11-13T01:30:51,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451570) roll requested 2024-11-13T01:30:51,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451576 2024-11-13T01:30:51,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451576 2024-11-13T01:30:51,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451576) roll requested 2024-11-13T01:30:51,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451583 2024-11-13T01:30:51,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451583 2024-11-13T01:30:51,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451583) roll requested 2024-11-13T01:30:51,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451591 2024-11-13T01:30:51,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451591 2024-11-13T01:30:51,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451591) roll requested 2024-11-13T01:30:51,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451601 2024-11-13T01:30:51,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451601 2024-11-13T01:30:51,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451601) roll requested 2024-11-13T01:30:51,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451609 2024-11-13T01:30:51,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451609 2024-11-13T01:30:51,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451609) roll requested 2024-11-13T01:30:51,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451619 2024-11-13T01:30:51,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451619 2024-11-13T01:30:51,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451619) roll requested 2024-11-13T01:30:51,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451626 2024-11-13T01:30:51,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451626 2024-11-13T01:30:51,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451626) roll requested 2024-11-13T01:30:51,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451636 2024-11-13T01:30:51,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451636 2024-11-13T01:30:51,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451636) roll requested 2024-11-13T01:30:51,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451646 2024-11-13T01:30:51,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451646 2024-11-13T01:30:51,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451646) roll requested 2024-11-13T01:30:51,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451655 2024-11-13T01:30:51,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451655 2024-11-13T01:30:51,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451655) roll requested 2024-11-13T01:30:51,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451664 2024-11-13T01:30:51,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451664 2024-11-13T01:30:51,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451664) roll requested 2024-11-13T01:30:51,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451671 2024-11-13T01:30:51,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451671 2024-11-13T01:30:51,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451671) roll requested 2024-11-13T01:30:51,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451677 2024-11-13T01:30:51,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451677 2024-11-13T01:30:51,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451677) roll requested 2024-11-13T01:30:51,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451685 2024-11-13T01:30:51,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451685 2024-11-13T01:30:51,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451685) roll requested 2024-11-13T01:30:51,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451694 2024-11-13T01:30:51,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451694 2024-11-13T01:30:51,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451694) roll requested 2024-11-13T01:30:51,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451706 2024-11-13T01:30:51,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451706 2024-11-13T01:30:51,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451706) roll requested 2024-11-13T01:30:51,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451720 2024-11-13T01:30:51,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451720 2024-11-13T01:30:51,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451720) roll requested 2024-11-13T01:30:51,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451727 2024-11-13T01:30:51,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451727 2024-11-13T01:30:51,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451727) roll requested 2024-11-13T01:30:51,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451734 2024-11-13T01:30:51,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451734 2024-11-13T01:30:51,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451734) roll requested 2024-11-13T01:30:51,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451745 2024-11-13T01:30:51,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451745 2024-11-13T01:30:51,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451745) roll requested 2024-11-13T01:30:51,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451757 2024-11-13T01:30:51,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451757 2024-11-13T01:30:51,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451757) roll requested 2024-11-13T01:30:51,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451763 2024-11-13T01:30:51,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451763 2024-11-13T01:30:51,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451763) roll requested 2024-11-13T01:30:51,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451771 2024-11-13T01:30:51,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451771 2024-11-13T01:30:51,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451771) roll requested 2024-11-13T01:30:51,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451778 2024-11-13T01:30:51,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451778 2024-11-13T01:30:51,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451778) roll requested 2024-11-13T01:30:51,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451784 2024-11-13T01:30:51,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451784 2024-11-13T01:30:51,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451784) roll requested 2024-11-13T01:30:51,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451790 2024-11-13T01:30:51,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451790 2024-11-13T01:30:51,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451790) roll requested 2024-11-13T01:30:51,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451800 2024-11-13T01:30:51,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,805 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,805 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451790 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451800 2024-11-13T01:30:51,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451800) roll requested 2024-11-13T01:30:51,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451806 2024-11-13T01:30:51,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451806 2024-11-13T01:30:51,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451806) roll requested 2024-11-13T01:30:51,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451816 2024-11-13T01:30:51,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451816 2024-11-13T01:30:51,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451816) roll requested 2024-11-13T01:30:51,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451822 2024-11-13T01:30:51,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451822 2024-11-13T01:30:51,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451822) roll requested 2024-11-13T01:30:51,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451829 2024-11-13T01:30:51,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451829 2024-11-13T01:30:51,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451829) roll requested 2024-11-13T01:30:51,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451835 2024-11-13T01:30:51,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451835 2024-11-13T01:30:51,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451835) roll requested 2024-11-13T01:30:51,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451841 2024-11-13T01:30:51,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451841 2024-11-13T01:30:51,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451841) roll requested 2024-11-13T01:30:51,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451848 2024-11-13T01:30:51,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451848 2024-11-13T01:30:51,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451848) roll requested 2024-11-13T01:30:51,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451855 2024-11-13T01:30:51,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451855 2024-11-13T01:30:51,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451855) roll requested 2024-11-13T01:30:51,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451868 2024-11-13T01:30:51,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451868 2024-11-13T01:30:51,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451868) roll requested 2024-11-13T01:30:51,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451875 2024-11-13T01:30:51,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451875 2024-11-13T01:30:51,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451875) roll requested 2024-11-13T01:30:51,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451882 2024-11-13T01:30:51,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451882 2024-11-13T01:30:51,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451882) roll requested 2024-11-13T01:30:51,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451889 2024-11-13T01:30:51,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451889 2024-11-13T01:30:51,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451889) roll requested 2024-11-13T01:30:51,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451901 2024-11-13T01:30:51,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451901 2024-11-13T01:30:51,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451901) roll requested 2024-11-13T01:30:51,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451912 2024-11-13T01:30:51,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451912 2024-11-13T01:30:51,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451912) roll requested 2024-11-13T01:30:51,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451919 2024-11-13T01:30:51,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451919 2024-11-13T01:30:51,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451919) roll requested 2024-11-13T01:30:51,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451928 2024-11-13T01:30:51,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451928 2024-11-13T01:30:51,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451928) roll requested 2024-11-13T01:30:51,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451936 2024-11-13T01:30:51,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451936 2024-11-13T01:30:51,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451936) roll requested 2024-11-13T01:30:51,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451944 2024-11-13T01:30:51,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451944 2024-11-13T01:30:51,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451944) roll requested 2024-11-13T01:30:51,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451952 2024-11-13T01:30:51,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451952 2024-11-13T01:30:51,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451952) roll requested 2024-11-13T01:30:51,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451960 2024-11-13T01:30:51,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451960 2024-11-13T01:30:51,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451960) roll requested 2024-11-13T01:30:51,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451967 2024-11-13T01:30:51,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451967 2024-11-13T01:30:51,973 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451967) roll requested 2024-11-13T01:30:51,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451973 2024-11-13T01:30:51,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451973 2024-11-13T01:30:51,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451973) roll requested 2024-11-13T01:30:51,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451979 2024-11-13T01:30:51,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451979 2024-11-13T01:30:51,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451979) roll requested 2024-11-13T01:30:51,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451986 2024-11-13T01:30:51,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:51,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451986 2024-11-13T01:30:51,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:51,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:51,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451986) roll requested 2024-11-13T01:30:51,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461451993 2024-11-13T01:30:52,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451993 2024-11-13T01:30:52,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461451993) roll requested 2024-11-13T01:30:52,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452001 2024-11-13T01:30:52,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461451993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452001 2024-11-13T01:30:52,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452001) roll requested 2024-11-13T01:30:52,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452007 2024-11-13T01:30:52,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452007 2024-11-13T01:30:52,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452007) roll requested 2024-11-13T01:30:52,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452013 2024-11-13T01:30:52,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452013 2024-11-13T01:30:52,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452013) roll requested 2024-11-13T01:30:52,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452020 2024-11-13T01:30:52,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452020 2024-11-13T01:30:52,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452020) roll requested 2024-11-13T01:30:52,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452027 2024-11-13T01:30:52,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452027 2024-11-13T01:30:52,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452027) roll requested 2024-11-13T01:30:52,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452040 2024-11-13T01:30:52,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452040 2024-11-13T01:30:52,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452040) roll requested 2024-11-13T01:30:52,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452047 2024-11-13T01:30:52,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452047 2024-11-13T01:30:52,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452047) roll requested 2024-11-13T01:30:52,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452054 2024-11-13T01:30:52,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452054 2024-11-13T01:30:52,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452054) roll requested 2024-11-13T01:30:52,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452062 2024-11-13T01:30:52,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452062 2024-11-13T01:30:52,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452062) roll requested 2024-11-13T01:30:52,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452070 2024-11-13T01:30:52,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452070 2024-11-13T01:30:52,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452070) roll requested 2024-11-13T01:30:52,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452079 2024-11-13T01:30:52,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452079 2024-11-13T01:30:52,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452079) roll requested 2024-11-13T01:30:52,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452089 2024-11-13T01:30:52,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452089 2024-11-13T01:30:52,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452089) roll requested 2024-11-13T01:30:52,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452095 2024-11-13T01:30:52,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452095 2024-11-13T01:30:52,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452095) roll requested 2024-11-13T01:30:52,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452101 2024-11-13T01:30:52,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452101 2024-11-13T01:30:52,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452101) roll requested 2024-11-13T01:30:52,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452109 2024-11-13T01:30:52,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452109 2024-11-13T01:30:52,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452109) roll requested 2024-11-13T01:30:52,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452115 2024-11-13T01:30:52,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452109 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452115 2024-11-13T01:30:52,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452115) roll requested 2024-11-13T01:30:52,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452121 2024-11-13T01:30:52,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452121 2024-11-13T01:30:52,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452121) roll requested 2024-11-13T01:30:52,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452128 2024-11-13T01:30:52,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452128 2024-11-13T01:30:52,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452128) roll requested 2024-11-13T01:30:52,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452134 2024-11-13T01:30:52,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452134 2024-11-13T01:30:52,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452134) roll requested 2024-11-13T01:30:52,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452139 2024-11-13T01:30:52,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452139 2024-11-13T01:30:52,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452139) roll requested 2024-11-13T01:30:52,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452145 2024-11-13T01:30:52,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452145 2024-11-13T01:30:52,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452145) roll requested 2024-11-13T01:30:52,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452153 2024-11-13T01:30:52,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452153 2024-11-13T01:30:52,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452153) roll requested 2024-11-13T01:30:52,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452161 2024-11-13T01:30:52,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452161 2024-11-13T01:30:52,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452161) roll requested 2024-11-13T01:30:52,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452169 2024-11-13T01:30:52,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452169 2024-11-13T01:30:52,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452169) roll requested 2024-11-13T01:30:52,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452178 2024-11-13T01:30:52,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452178 2024-11-13T01:30:52,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452178) roll requested 2024-11-13T01:30:52,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452187 2024-11-13T01:30:52,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452187 2024-11-13T01:30:52,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452187) roll requested 2024-11-13T01:30:52,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452195 2024-11-13T01:30:52,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452195 2024-11-13T01:30:52,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452195) roll requested 2024-11-13T01:30:52,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452203 2024-11-13T01:30:52,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452203 2024-11-13T01:30:52,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452203) roll requested 2024-11-13T01:30:52,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452213 2024-11-13T01:30:52,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452213 2024-11-13T01:30:52,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452213) roll requested 2024-11-13T01:30:52,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452220 2024-11-13T01:30:52,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452220 2024-11-13T01:30:52,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452220) roll requested 2024-11-13T01:30:52,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452229 2024-11-13T01:30:52,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452229 2024-11-13T01:30:52,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452229) roll requested 2024-11-13T01:30:52,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452236 2024-11-13T01:30:52,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452236 2024-11-13T01:30:52,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452236) roll requested 2024-11-13T01:30:52,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452245 2024-11-13T01:30:52,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452245 2024-11-13T01:30:52,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452245) roll requested 2024-11-13T01:30:52,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452251 2024-11-13T01:30:52,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452251 2024-11-13T01:30:52,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452251) roll requested 2024-11-13T01:30:52,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452260 2024-11-13T01:30:52,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452260 2024-11-13T01:30:52,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452260) roll requested 2024-11-13T01:30:52,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452266 2024-11-13T01:30:52,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452266 2024-11-13T01:30:52,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452266) roll requested 2024-11-13T01:30:52,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452273 2024-11-13T01:30:52,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452266 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452273 2024-11-13T01:30:52,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452273) roll requested 2024-11-13T01:30:52,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452279 2024-11-13T01:30:52,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452279 2024-11-13T01:30:52,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452279) roll requested 2024-11-13T01:30:52,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452284 2024-11-13T01:30:52,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452284 2024-11-13T01:30:52,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452284) roll requested 2024-11-13T01:30:52,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452291 2024-11-13T01:30:52,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452291 2024-11-13T01:30:52,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452291) roll requested 2024-11-13T01:30:52,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452299 2024-11-13T01:30:52,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452299 2024-11-13T01:30:52,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452299) roll requested 2024-11-13T01:30:52,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452307 2024-11-13T01:30:52,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452307 2024-11-13T01:30:52,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452307) roll requested 2024-11-13T01:30:52,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452317 2024-11-13T01:30:52,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452317 2024-11-13T01:30:52,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452317) roll requested 2024-11-13T01:30:52,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452327 2024-11-13T01:30:52,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452327 2024-11-13T01:30:52,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452327) roll requested 2024-11-13T01:30:52,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452334 2024-11-13T01:30:52,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452334 2024-11-13T01:30:52,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452334) roll requested 2024-11-13T01:30:52,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452340 2024-11-13T01:30:52,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452340 2024-11-13T01:30:52,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452340) roll requested 2024-11-13T01:30:52,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452347 2024-11-13T01:30:52,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452347 2024-11-13T01:30:52,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,353 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452347) roll requested 2024-11-13T01:30:52,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452354 2024-11-13T01:30:52,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452354 2024-11-13T01:30:52,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452354) roll requested 2024-11-13T01:30:52,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452362 2024-11-13T01:30:52,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452362 2024-11-13T01:30:52,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452362) roll requested 2024-11-13T01:30:52,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452371 2024-11-13T01:30:52,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452371 2024-11-13T01:30:52,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452371) roll requested 2024-11-13T01:30:52,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452380 2024-11-13T01:30:52,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452380 2024-11-13T01:30:52,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452380) roll requested 2024-11-13T01:30:52,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452389 2024-11-13T01:30:52,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452389 2024-11-13T01:30:52,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452389) roll requested 2024-11-13T01:30:52,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452397 2024-11-13T01:30:52,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452397 2024-11-13T01:30:52,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452397) roll requested 2024-11-13T01:30:52,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452409 2024-11-13T01:30:52,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452409 2024-11-13T01:30:52,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452409) roll requested 2024-11-13T01:30:52,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452416 2024-11-13T01:30:52,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452416 2024-11-13T01:30:52,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452416) roll requested 2024-11-13T01:30:52,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452422 2024-11-13T01:30:52,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452422 2024-11-13T01:30:52,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452422) roll requested 2024-11-13T01:30:52,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452428 2024-11-13T01:30:52,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452428 2024-11-13T01:30:52,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452428) roll requested 2024-11-13T01:30:52,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452434 2024-11-13T01:30:52,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452434 2024-11-13T01:30:52,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452434) roll requested 2024-11-13T01:30:52,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452441 2024-11-13T01:30:52,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452441 2024-11-13T01:30:52,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452441) roll requested 2024-11-13T01:30:52,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452450 2024-11-13T01:30:52,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452450 2024-11-13T01:30:52,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452450) roll requested 2024-11-13T01:30:52,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452460 2024-11-13T01:30:52,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452460 2024-11-13T01:30:52,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452460) roll requested 2024-11-13T01:30:52,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452469 2024-11-13T01:30:52,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452460 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452469 2024-11-13T01:30:52,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452469) roll requested 2024-11-13T01:30:52,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452476 2024-11-13T01:30:52,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452476 2024-11-13T01:30:52,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452476) roll requested 2024-11-13T01:30:52,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452482 2024-11-13T01:30:52,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452482 2024-11-13T01:30:52,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452482) roll requested 2024-11-13T01:30:52,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452492 2024-11-13T01:30:52,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452492 2024-11-13T01:30:52,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452492) roll requested 2024-11-13T01:30:52,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452500 2024-11-13T01:30:52,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452500 2024-11-13T01:30:52,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452500) roll requested 2024-11-13T01:30:52,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452506 2024-11-13T01:30:52,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452506 2024-11-13T01:30:52,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452506) roll requested 2024-11-13T01:30:52,512 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452512 2024-11-13T01:30:52,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452512 2024-11-13T01:30:52,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452512) roll requested 2024-11-13T01:30:52,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452519 2024-11-13T01:30:52,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452512 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452519 2024-11-13T01:30:52,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452519) roll requested 2024-11-13T01:30:52,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452527 2024-11-13T01:30:52,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452527 2024-11-13T01:30:52,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452527) roll requested 2024-11-13T01:30:52,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452535 2024-11-13T01:30:52,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452535 2024-11-13T01:30:52,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452535) roll requested 2024-11-13T01:30:52,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452543 2024-11-13T01:30:52,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452543 2024-11-13T01:30:52,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452543) roll requested 2024-11-13T01:30:52,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452551 2024-11-13T01:30:52,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452551 2024-11-13T01:30:52,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452551) roll requested 2024-11-13T01:30:52,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452557 2024-11-13T01:30:52,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452557 2024-11-13T01:30:52,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452557) roll requested 2024-11-13T01:30:52,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452562 2024-11-13T01:30:52,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452562 2024-11-13T01:30:52,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452562) roll requested 2024-11-13T01:30:52,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452570 2024-11-13T01:30:52,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452570 2024-11-13T01:30:52,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452570) roll requested 2024-11-13T01:30:52,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452577 2024-11-13T01:30:52,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452577 2024-11-13T01:30:52,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452577) roll requested 2024-11-13T01:30:52,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452584 2024-11-13T01:30:52,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452584 2024-11-13T01:30:52,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452584) roll requested 2024-11-13T01:30:52,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452593 2024-11-13T01:30:52,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452593 2024-11-13T01:30:52,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452593) roll requested 2024-11-13T01:30:52,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452603 2024-11-13T01:30:52,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452603 2024-11-13T01:30:52,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452603) roll requested 2024-11-13T01:30:52,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452613 2024-11-13T01:30:52,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452613 2024-11-13T01:30:52,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452613) roll requested 2024-11-13T01:30:52,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452622 2024-11-13T01:30:52,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452622 2024-11-13T01:30:52,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452622) roll requested 2024-11-13T01:30:52,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452632 2024-11-13T01:30:52,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452632 2024-11-13T01:30:52,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452632) roll requested 2024-11-13T01:30:52,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452639 2024-11-13T01:30:52,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452639 2024-11-13T01:30:52,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452639) roll requested 2024-11-13T01:30:52,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452648 2024-11-13T01:30:52,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452648 2024-11-13T01:30:52,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452648) roll requested 2024-11-13T01:30:52,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452656 2024-11-13T01:30:52,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452656 2024-11-13T01:30:52,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452656) roll requested 2024-11-13T01:30:52,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452665 2024-11-13T01:30:52,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452665 2024-11-13T01:30:52,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452665) roll requested 2024-11-13T01:30:52,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452676 2024-11-13T01:30:52,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452676 2024-11-13T01:30:52,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452676) roll requested 2024-11-13T01:30:52,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452687 2024-11-13T01:30:52,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452687 2024-11-13T01:30:52,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452687) roll requested 2024-11-13T01:30:52,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452697 2024-11-13T01:30:52,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452687 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452697 2024-11-13T01:30:52,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452697) roll requested 2024-11-13T01:30:52,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452707 2024-11-13T01:30:52,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452707 2024-11-13T01:30:52,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452707) roll requested 2024-11-13T01:30:52,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452721 2024-11-13T01:30:52,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452721 2024-11-13T01:30:52,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452721) roll requested 2024-11-13T01:30:52,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452730 2024-11-13T01:30:52,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452730 2024-11-13T01:30:52,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452730) roll requested 2024-11-13T01:30:52,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452736 2024-11-13T01:30:52,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452736 2024-11-13T01:30:52,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452736) roll requested 2024-11-13T01:30:52,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452741 2024-11-13T01:30:52,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452741 2024-11-13T01:30:52,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452741) roll requested 2024-11-13T01:30:52,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452748 2024-11-13T01:30:52,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452748 2024-11-13T01:30:52,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452748) roll requested 2024-11-13T01:30:52,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452755 2024-11-13T01:30:52,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452755 2024-11-13T01:30:52,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452755) roll requested 2024-11-13T01:30:52,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452764 2024-11-13T01:30:52,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452764 2024-11-13T01:30:52,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452764) roll requested 2024-11-13T01:30:52,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452778 2024-11-13T01:30:52,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452778 2024-11-13T01:30:52,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452778) roll requested 2024-11-13T01:30:52,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452788 2024-11-13T01:30:52,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452788 2024-11-13T01:30:52,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452788) roll requested 2024-11-13T01:30:52,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452797 2024-11-13T01:30:52,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452797 2024-11-13T01:30:52,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452797) roll requested 2024-11-13T01:30:52,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452803 2024-11-13T01:30:52,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452803 2024-11-13T01:30:52,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452803) roll requested 2024-11-13T01:30:52,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452811 2024-11-13T01:30:52,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452811 2024-11-13T01:30:52,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452811) roll requested 2024-11-13T01:30:52,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452820 2024-11-13T01:30:52,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452820 2024-11-13T01:30:52,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452820) roll requested 2024-11-13T01:30:52,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452828 2024-11-13T01:30:52,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452828 2024-11-13T01:30:52,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452828) roll requested 2024-11-13T01:30:52,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452834 2024-11-13T01:30:52,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452834 2024-11-13T01:30:52,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452834) roll requested 2024-11-13T01:30:52,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452839 2024-11-13T01:30:52,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452839 2024-11-13T01:30:52,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452839) roll requested 2024-11-13T01:30:52,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452846 2024-11-13T01:30:52,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452846 2024-11-13T01:30:52,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452846) roll requested 2024-11-13T01:30:52,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452853 2024-11-13T01:30:52,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452853 2024-11-13T01:30:52,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452853) roll requested 2024-11-13T01:30:52,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452860 2024-11-13T01:30:52,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452860 2024-11-13T01:30:52,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452860) roll requested 2024-11-13T01:30:52,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452866 2024-11-13T01:30:52,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452866 2024-11-13T01:30:52,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452866) roll requested 2024-11-13T01:30:52,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452872 2024-11-13T01:30:52,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452872 2024-11-13T01:30:52,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452872) roll requested 2024-11-13T01:30:52,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452881 2024-11-13T01:30:52,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452881 2024-11-13T01:30:52,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452881) roll requested 2024-11-13T01:30:52,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452887 2024-11-13T01:30:52,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452887 2024-11-13T01:30:52,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452887) roll requested 2024-11-13T01:30:52,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452893 2024-11-13T01:30:52,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452893 2024-11-13T01:30:52,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452893) roll requested 2024-11-13T01:30:52,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452900 2024-11-13T01:30:52,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452900 2024-11-13T01:30:52,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452900) roll requested 2024-11-13T01:30:52,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452909 2024-11-13T01:30:52,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452909 2024-11-13T01:30:52,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452909) roll requested 2024-11-13T01:30:52,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452918 2024-11-13T01:30:52,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452918 2024-11-13T01:30:52,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452918) roll requested 2024-11-13T01:30:52,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452927 2024-11-13T01:30:52,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452927 2024-11-13T01:30:52,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452927) roll requested 2024-11-13T01:30:52,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452938 2024-11-13T01:30:52,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452938 2024-11-13T01:30:52,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452938) roll requested 2024-11-13T01:30:52,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452949 2024-11-13T01:30:52,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452949 2024-11-13T01:30:52,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452949) roll requested 2024-11-13T01:30:52,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452958 2024-11-13T01:30:52,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452958 2024-11-13T01:30:52,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452958) roll requested 2024-11-13T01:30:52,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452969 2024-11-13T01:30:52,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452969 2024-11-13T01:30:52,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452969) roll requested 2024-11-13T01:30:52,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452976 2024-11-13T01:30:52,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452976 2024-11-13T01:30:52,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452976) roll requested 2024-11-13T01:30:52,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452983 2024-11-13T01:30:52,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452983 2024-11-13T01:30:52,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452983) roll requested 2024-11-13T01:30:52,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452991 2024-11-13T01:30:52,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:52,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452991 2024-11-13T01:30:52,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:52,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:52,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452991) roll requested 2024-11-13T01:30:52,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461452999 2024-11-13T01:30:53,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452999 2024-11-13T01:30:53,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461452999) roll requested 2024-11-13T01:30:53,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453008 2024-11-13T01:30:53,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461452999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453008 2024-11-13T01:30:53,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453008) roll requested 2024-11-13T01:30:53,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453017 2024-11-13T01:30:53,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453017 2024-11-13T01:30:53,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453017) roll requested 2024-11-13T01:30:53,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453026 2024-11-13T01:30:53,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453026 2024-11-13T01:30:53,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453026) roll requested 2024-11-13T01:30:53,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453035 2024-11-13T01:30:53,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453035 2024-11-13T01:30:53,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,049 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453035) roll requested 2024-11-13T01:30:53,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453050 2024-11-13T01:30:53,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453050 2024-11-13T01:30:53,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453050) roll requested 2024-11-13T01:30:53,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453056 2024-11-13T01:30:53,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453056 2024-11-13T01:30:53,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453056) roll requested 2024-11-13T01:30:53,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453062 2024-11-13T01:30:53,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453062 2024-11-13T01:30:53,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453062) roll requested 2024-11-13T01:30:53,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453068 2024-11-13T01:30:53,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453068 2024-11-13T01:30:53,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453068) roll requested 2024-11-13T01:30:53,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453074 2024-11-13T01:30:53,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453074 2024-11-13T01:30:53,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453074) roll requested 2024-11-13T01:30:53,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453083 2024-11-13T01:30:53,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453083 2024-11-13T01:30:53,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453083) roll requested 2024-11-13T01:30:53,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453090 2024-11-13T01:30:53,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453090 2024-11-13T01:30:53,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453090) roll requested 2024-11-13T01:30:53,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453099 2024-11-13T01:30:53,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453099 2024-11-13T01:30:53,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453099) roll requested 2024-11-13T01:30:53,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453107 2024-11-13T01:30:53,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453107 2024-11-13T01:30:53,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453107) roll requested 2024-11-13T01:30:53,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453118 2024-11-13T01:30:53,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453118 2024-11-13T01:30:53,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453118) roll requested 2024-11-13T01:30:53,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453126 2024-11-13T01:30:53,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453126 2024-11-13T01:30:53,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453126) roll requested 2024-11-13T01:30:53,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453135 2024-11-13T01:30:53,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453135 2024-11-13T01:30:53,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453135) roll requested 2024-11-13T01:30:53,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453140 2024-11-13T01:30:53,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453140 2024-11-13T01:30:53,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453140) roll requested 2024-11-13T01:30:53,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453146 2024-11-13T01:30:53,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453146 2024-11-13T01:30:53,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,151 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453146) roll requested 2024-11-13T01:30:53,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453151 2024-11-13T01:30:53,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453151 2024-11-13T01:30:53,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453151) roll requested 2024-11-13T01:30:53,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453157 2024-11-13T01:30:53,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453157 2024-11-13T01:30:53,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453157) roll requested 2024-11-13T01:30:53,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453163 2024-11-13T01:30:53,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453163 2024-11-13T01:30:53,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453163) roll requested 2024-11-13T01:30:53,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453170 2024-11-13T01:30:53,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453170 2024-11-13T01:30:53,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453170) roll requested 2024-11-13T01:30:53,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453175 2024-11-13T01:30:53,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453175 2024-11-13T01:30:53,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453175) roll requested 2024-11-13T01:30:53,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453182 2024-11-13T01:30:53,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453182 2024-11-13T01:30:53,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453182) roll requested 2024-11-13T01:30:53,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453192 2024-11-13T01:30:53,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453192 2024-11-13T01:30:53,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453192) roll requested 2024-11-13T01:30:53,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453200 2024-11-13T01:30:53,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453200 2024-11-13T01:30:53,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453200) roll requested 2024-11-13T01:30:53,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453208 2024-11-13T01:30:53,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453208 2024-11-13T01:30:53,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453208) roll requested 2024-11-13T01:30:53,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453214 2024-11-13T01:30:53,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453214 2024-11-13T01:30:53,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453214) roll requested 2024-11-13T01:30:53,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453220 2024-11-13T01:30:53,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453220 2024-11-13T01:30:53,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453220) roll requested 2024-11-13T01:30:53,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453226 2024-11-13T01:30:53,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453226 2024-11-13T01:30:53,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453226) roll requested 2024-11-13T01:30:53,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453233 2024-11-13T01:30:53,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453233 2024-11-13T01:30:53,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453233) roll requested 2024-11-13T01:30:53,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453239 2024-11-13T01:30:53,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453239 2024-11-13T01:30:53,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453239) roll requested 2024-11-13T01:30:53,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453245 2024-11-13T01:30:53,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453245 2024-11-13T01:30:53,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453245) roll requested 2024-11-13T01:30:53,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453251 2024-11-13T01:30:53,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453251 2024-11-13T01:30:53,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453251) roll requested 2024-11-13T01:30:53,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453257 2024-11-13T01:30:53,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453257 2024-11-13T01:30:53,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453257) roll requested 2024-11-13T01:30:53,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453263 2024-11-13T01:30:53,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453263 2024-11-13T01:30:53,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453263) roll requested 2024-11-13T01:30:53,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453268 2024-11-13T01:30:53,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453268 2024-11-13T01:30:53,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453268) roll requested 2024-11-13T01:30:53,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453275 2024-11-13T01:30:53,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453275 2024-11-13T01:30:53,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453275) roll requested 2024-11-13T01:30:53,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453282 2024-11-13T01:30:53,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453282 2024-11-13T01:30:53,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453282) roll requested 2024-11-13T01:30:53,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453289 2024-11-13T01:30:53,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453289 2024-11-13T01:30:53,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453289) roll requested 2024-11-13T01:30:53,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453297 2024-11-13T01:30:53,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453297 2024-11-13T01:30:53,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453297) roll requested 2024-11-13T01:30:53,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453304 2024-11-13T01:30:53,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453304 2024-11-13T01:30:53,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453304) roll requested 2024-11-13T01:30:53,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453312 2024-11-13T01:30:53,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453312 2024-11-13T01:30:53,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453312) roll requested 2024-11-13T01:30:53,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453318 2024-11-13T01:30:53,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453318 2024-11-13T01:30:53,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453318) roll requested 2024-11-13T01:30:53,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453324 2024-11-13T01:30:53,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453324 2024-11-13T01:30:53,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453324) roll requested 2024-11-13T01:30:53,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453330 2024-11-13T01:30:53,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453330 2024-11-13T01:30:53,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453330) roll requested 2024-11-13T01:30:53,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453336 2024-11-13T01:30:53,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453336 2024-11-13T01:30:53,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453336) roll requested 2024-11-13T01:30:53,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453342 2024-11-13T01:30:53,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453342 2024-11-13T01:30:53,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453342) roll requested 2024-11-13T01:30:53,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453347 2024-11-13T01:30:53,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453347 2024-11-13T01:30:53,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453347) roll requested 2024-11-13T01:30:53,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453354 2024-11-13T01:30:53,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453354 2024-11-13T01:30:53,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,361 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453354) roll requested 2024-11-13T01:30:53,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453361 2024-11-13T01:30:53,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453354 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453361 2024-11-13T01:30:53,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453361) roll requested 2024-11-13T01:30:53,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453368 2024-11-13T01:30:53,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453368 2024-11-13T01:30:53,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453368) roll requested 2024-11-13T01:30:53,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453376 2024-11-13T01:30:53,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453376 2024-11-13T01:30:53,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453376) roll requested 2024-11-13T01:30:53,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453385 2024-11-13T01:30:53,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453385 2024-11-13T01:30:53,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453385) roll requested 2024-11-13T01:30:53,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453393 2024-11-13T01:30:53,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453393 2024-11-13T01:30:53,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453393) roll requested 2024-11-13T01:30:53,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453400 2024-11-13T01:30:53,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453400 2024-11-13T01:30:53,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453400) roll requested 2024-11-13T01:30:53,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453406 2024-11-13T01:30:53,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453406 2024-11-13T01:30:53,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453406) roll requested 2024-11-13T01:30:53,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453412 2024-11-13T01:30:53,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453412 2024-11-13T01:30:53,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453412) roll requested 2024-11-13T01:30:53,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453419 2024-11-13T01:30:53,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453419 2024-11-13T01:30:53,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453419) roll requested 2024-11-13T01:30:53,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453424 2024-11-13T01:30:53,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,429 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,429 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453424 2024-11-13T01:30:53,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,430 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,430 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453424) roll requested 2024-11-13T01:30:53,430 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453430 2024-11-13T01:30:53,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453430 2024-11-13T01:30:53,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453430) roll requested 2024-11-13T01:30:53,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453439 2024-11-13T01:30:53,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453430 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453439 2024-11-13T01:30:53,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453439) roll requested 2024-11-13T01:30:53,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453446 2024-11-13T01:30:53,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453446 2024-11-13T01:30:53,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453446) roll requested 2024-11-13T01:30:53,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453451 2024-11-13T01:30:53,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453451 2024-11-13T01:30:53,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453451) roll requested 2024-11-13T01:30:53,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453457 2024-11-13T01:30:53,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453457 2024-11-13T01:30:53,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453457) roll requested 2024-11-13T01:30:53,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453465 2024-11-13T01:30:53,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453465 2024-11-13T01:30:53,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453465) roll requested 2024-11-13T01:30:53,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453471 2024-11-13T01:30:53,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453471 2024-11-13T01:30:53,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453471) roll requested 2024-11-13T01:30:53,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453478 2024-11-13T01:30:53,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453478 2024-11-13T01:30:53,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453478) roll requested 2024-11-13T01:30:53,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453484 2024-11-13T01:30:53,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453484 2024-11-13T01:30:53,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453484) roll requested 2024-11-13T01:30:53,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453490 2024-11-13T01:30:53,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453490 2024-11-13T01:30:53,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453490) roll requested 2024-11-13T01:30:53,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453495 2024-11-13T01:30:53,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453495 2024-11-13T01:30:53,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453495) roll requested 2024-11-13T01:30:53,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453500 2024-11-13T01:30:53,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453500 2024-11-13T01:30:53,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453500) roll requested 2024-11-13T01:30:53,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453506 2024-11-13T01:30:53,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453506 2024-11-13T01:30:53,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453506) roll requested 2024-11-13T01:30:53,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453513 2024-11-13T01:30:53,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453513 2024-11-13T01:30:53,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453513) roll requested 2024-11-13T01:30:53,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453520 2024-11-13T01:30:53,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453520 2024-11-13T01:30:53,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453520) roll requested 2024-11-13T01:30:53,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453530 2024-11-13T01:30:53,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453530 2024-11-13T01:30:53,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453530) roll requested 2024-11-13T01:30:53,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453537 2024-11-13T01:30:53,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453537 2024-11-13T01:30:53,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453537) roll requested 2024-11-13T01:30:53,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453544 2024-11-13T01:30:53,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453544 2024-11-13T01:30:53,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453544) roll requested 2024-11-13T01:30:53,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453551 2024-11-13T01:30:53,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453551 2024-11-13T01:30:53,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,558 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453551) roll requested 2024-11-13T01:30:53,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453558 2024-11-13T01:30:53,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453558 2024-11-13T01:30:53,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453558) roll requested 2024-11-13T01:30:53,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453565 2024-11-13T01:30:53,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453565 2024-11-13T01:30:53,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453565) roll requested 2024-11-13T01:30:53,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453573 2024-11-13T01:30:53,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453573 2024-11-13T01:30:53,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453573) roll requested 2024-11-13T01:30:53,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453580 2024-11-13T01:30:53,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453580 2024-11-13T01:30:53,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453580) roll requested 2024-11-13T01:30:53,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453588 2024-11-13T01:30:53,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453588 2024-11-13T01:30:53,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453588) roll requested 2024-11-13T01:30:53,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453597 2024-11-13T01:30:53,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453597 2024-11-13T01:30:53,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453597) roll requested 2024-11-13T01:30:53,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453603 2024-11-13T01:30:53,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453603 2024-11-13T01:30:53,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453603) roll requested 2024-11-13T01:30:53,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453609 2024-11-13T01:30:53,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453609 2024-11-13T01:30:53,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453609) roll requested 2024-11-13T01:30:53,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453616 2024-11-13T01:30:53,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453616 2024-11-13T01:30:53,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453616) roll requested 2024-11-13T01:30:53,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453621 2024-11-13T01:30:53,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453621 2024-11-13T01:30:53,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453621) roll requested 2024-11-13T01:30:53,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453626 2024-11-13T01:30:53,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453626 2024-11-13T01:30:53,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453626) roll requested 2024-11-13T01:30:53,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453633 2024-11-13T01:30:53,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453633 2024-11-13T01:30:53,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453633) roll requested 2024-11-13T01:30:53,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453639 2024-11-13T01:30:53,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453639 2024-11-13T01:30:53,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453639) roll requested 2024-11-13T01:30:53,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453646 2024-11-13T01:30:53,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453646 2024-11-13T01:30:53,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453646) roll requested 2024-11-13T01:30:53,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453654 2024-11-13T01:30:53,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453654 2024-11-13T01:30:53,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453654) roll requested 2024-11-13T01:30:53,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453663 2024-11-13T01:30:53,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453663 2024-11-13T01:30:53,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453663) roll requested 2024-11-13T01:30:53,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453676 2024-11-13T01:30:53,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453676 2024-11-13T01:30:53,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453676) roll requested 2024-11-13T01:30:53,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453683 2024-11-13T01:30:53,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453683 2024-11-13T01:30:53,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453683) roll requested 2024-11-13T01:30:53,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453693 2024-11-13T01:30:53,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453693 2024-11-13T01:30:53,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453693) roll requested 2024-11-13T01:30:53,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453702 2024-11-13T01:30:53,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453702 2024-11-13T01:30:53,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453702) roll requested 2024-11-13T01:30:53,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453713 2024-11-13T01:30:53,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453713 2024-11-13T01:30:53,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453713) roll requested 2024-11-13T01:30:53,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453720 2024-11-13T01:30:53,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453720 2024-11-13T01:30:53,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453720) roll requested 2024-11-13T01:30:53,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453726 2024-11-13T01:30:53,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453726 2024-11-13T01:30:53,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453726) roll requested 2024-11-13T01:30:53,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453731 2024-11-13T01:30:53,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453731 2024-11-13T01:30:53,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453731) roll requested 2024-11-13T01:30:53,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453741 2024-11-13T01:30:53,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453741 2024-11-13T01:30:53,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453741) roll requested 2024-11-13T01:30:53,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453747 2024-11-13T01:30:53,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453747 2024-11-13T01:30:53,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453747) roll requested 2024-11-13T01:30:53,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453756 2024-11-13T01:30:53,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453756 2024-11-13T01:30:53,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453756) roll requested 2024-11-13T01:30:53,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453762 2024-11-13T01:30:53,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453762 2024-11-13T01:30:53,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453762) roll requested 2024-11-13T01:30:53,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453767 2024-11-13T01:30:53,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453767 2024-11-13T01:30:53,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453767) roll requested 2024-11-13T01:30:53,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453772 2024-11-13T01:30:53,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453772 2024-11-13T01:30:53,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453772) roll requested 2024-11-13T01:30:53,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453780 2024-11-13T01:30:53,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453780 2024-11-13T01:30:53,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453780) roll requested 2024-11-13T01:30:53,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453788 2024-11-13T01:30:53,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453788 2024-11-13T01:30:53,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453788) roll requested 2024-11-13T01:30:53,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453796 2024-11-13T01:30:53,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453796 2024-11-13T01:30:53,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453796) roll requested 2024-11-13T01:30:53,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453802 2024-11-13T01:30:53,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453802 2024-11-13T01:30:53,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453802) roll requested 2024-11-13T01:30:53,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453807 2024-11-13T01:30:53,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453807 2024-11-13T01:30:53,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453807) roll requested 2024-11-13T01:30:53,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453812 2024-11-13T01:30:53,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453812 2024-11-13T01:30:53,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453812) roll requested 2024-11-13T01:30:53,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453819 2024-11-13T01:30:53,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453819 2024-11-13T01:30:53,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453819) roll requested 2024-11-13T01:30:53,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453826 2024-11-13T01:30:53,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453826 2024-11-13T01:30:53,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453826) roll requested 2024-11-13T01:30:53,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453835 2024-11-13T01:30:53,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453835 2024-11-13T01:30:53,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453835) roll requested 2024-11-13T01:30:53,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453840 2024-11-13T01:30:53,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453840 2024-11-13T01:30:53,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453840) roll requested 2024-11-13T01:30:53,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453846 2024-11-13T01:30:53,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453846 2024-11-13T01:30:53,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453846) roll requested 2024-11-13T01:30:53,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453852 2024-11-13T01:30:53,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453852 2024-11-13T01:30:53,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453852) roll requested 2024-11-13T01:30:53,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453859 2024-11-13T01:30:53,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453859 2024-11-13T01:30:53,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453859) roll requested 2024-11-13T01:30:53,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453866 2024-11-13T01:30:53,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453866 2024-11-13T01:30:53,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453866) roll requested 2024-11-13T01:30:53,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453874 2024-11-13T01:30:53,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453874 2024-11-13T01:30:53,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453874) roll requested 2024-11-13T01:30:53,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453884 2024-11-13T01:30:53,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453884 2024-11-13T01:30:53,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453884) roll requested 2024-11-13T01:30:53,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453891 2024-11-13T01:30:53,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453891 2024-11-13T01:30:53,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,897 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453891) roll requested 2024-11-13T01:30:53,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453897 2024-11-13T01:30:53,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453897 2024-11-13T01:30:53,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453897) roll requested 2024-11-13T01:30:53,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453903 2024-11-13T01:30:53,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453903 2024-11-13T01:30:53,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453903) roll requested 2024-11-13T01:30:53,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453908 2024-11-13T01:30:53,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453908 2024-11-13T01:30:53,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453908) roll requested 2024-11-13T01:30:53,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453914 2024-11-13T01:30:53,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453914 2024-11-13T01:30:53,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453914) roll requested 2024-11-13T01:30:53,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453919 2024-11-13T01:30:53,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453919 2024-11-13T01:30:53,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453919) roll requested 2024-11-13T01:30:53,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453925 2024-11-13T01:30:53,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453925 2024-11-13T01:30:53,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453925) roll requested 2024-11-13T01:30:53,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453931 2024-11-13T01:30:53,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453931 2024-11-13T01:30:53,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453931) roll requested 2024-11-13T01:30:53,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453936 2024-11-13T01:30:53,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453936 2024-11-13T01:30:53,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453936) roll requested 2024-11-13T01:30:53,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453943 2024-11-13T01:30:53,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453943 2024-11-13T01:30:53,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453943) roll requested 2024-11-13T01:30:53,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453950 2024-11-13T01:30:53,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453950 2024-11-13T01:30:53,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453950) roll requested 2024-11-13T01:30:53,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453957 2024-11-13T01:30:53,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453957 2024-11-13T01:30:53,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453957) roll requested 2024-11-13T01:30:53,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453966 2024-11-13T01:30:53,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453966 2024-11-13T01:30:53,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453966) roll requested 2024-11-13T01:30:53,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453975 2024-11-13T01:30:53,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453975 2024-11-13T01:30:53,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453975) roll requested 2024-11-13T01:30:53,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453981 2024-11-13T01:30:53,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453975 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453981 2024-11-13T01:30:53,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453981) roll requested 2024-11-13T01:30:53,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453988 2024-11-13T01:30:53,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:53,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453988 2024-11-13T01:30:53,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:53,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:53,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453988) roll requested 2024-11-13T01:30:53,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461453997 2024-11-13T01:30:54,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453997 2024-11-13T01:30:54,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461453997) roll requested 2024-11-13T01:30:54,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454005 2024-11-13T01:30:54,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461453997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454005 2024-11-13T01:30:54,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454005) roll requested 2024-11-13T01:30:54,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454011 2024-11-13T01:30:54,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454011 2024-11-13T01:30:54,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454011) roll requested 2024-11-13T01:30:54,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454018 2024-11-13T01:30:54,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454018 2024-11-13T01:30:54,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454018) roll requested 2024-11-13T01:30:54,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454027 2024-11-13T01:30:54,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454027 2024-11-13T01:30:54,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454027) roll requested 2024-11-13T01:30:54,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454035 2024-11-13T01:30:54,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454027 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454035 2024-11-13T01:30:54,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454035) roll requested 2024-11-13T01:30:54,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454044 2024-11-13T01:30:54,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454044 2024-11-13T01:30:54,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454044) roll requested 2024-11-13T01:30:54,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454052 2024-11-13T01:30:54,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454052 2024-11-13T01:30:54,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454052) roll requested 2024-11-13T01:30:54,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454057 2024-11-13T01:30:54,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454057 2024-11-13T01:30:54,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454057) roll requested 2024-11-13T01:30:54,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454063 2024-11-13T01:30:54,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454063 2024-11-13T01:30:54,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454063) roll requested 2024-11-13T01:30:54,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454070 2024-11-13T01:30:54,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454070 2024-11-13T01:30:54,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454070) roll requested 2024-11-13T01:30:54,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454077 2024-11-13T01:30:54,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454077 2024-11-13T01:30:54,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454077) roll requested 2024-11-13T01:30:54,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454084 2024-11-13T01:30:54,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454084 2024-11-13T01:30:54,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454084) roll requested 2024-11-13T01:30:54,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454089 2024-11-13T01:30:54,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454089 2024-11-13T01:30:54,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454089) roll requested 2024-11-13T01:30:54,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454095 2024-11-13T01:30:54,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454095 2024-11-13T01:30:54,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454095) roll requested 2024-11-13T01:30:54,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454101 2024-11-13T01:30:54,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454101 2024-11-13T01:30:54,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454101) roll requested 2024-11-13T01:30:54,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454108 2024-11-13T01:30:54,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454108 2024-11-13T01:30:54,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454108) roll requested 2024-11-13T01:30:54,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454116 2024-11-13T01:30:54,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454116 2024-11-13T01:30:54,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454116) roll requested 2024-11-13T01:30:54,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454125 2024-11-13T01:30:54,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454125 2024-11-13T01:30:54,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454125) roll requested 2024-11-13T01:30:54,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454132 2024-11-13T01:30:54,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454132 2024-11-13T01:30:54,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,139 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454132) roll requested 2024-11-13T01:30:54,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454139 2024-11-13T01:30:54,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454139 2024-11-13T01:30:54,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454139) roll requested 2024-11-13T01:30:54,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454148 2024-11-13T01:30:54,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454148 2024-11-13T01:30:54,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454148) roll requested 2024-11-13T01:30:54,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454156 2024-11-13T01:30:54,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454156 2024-11-13T01:30:54,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454156) roll requested 2024-11-13T01:30:54,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454164 2024-11-13T01:30:54,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454164 2024-11-13T01:30:54,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454164) roll requested 2024-11-13T01:30:54,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454172 2024-11-13T01:30:54,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454172 2024-11-13T01:30:54,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454172) roll requested 2024-11-13T01:30:54,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454180 2024-11-13T01:30:54,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454180 2024-11-13T01:30:54,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454180) roll requested 2024-11-13T01:30:54,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454189 2024-11-13T01:30:54,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454189 2024-11-13T01:30:54,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454189) roll requested 2024-11-13T01:30:54,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454195 2024-11-13T01:30:54,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454195 2024-11-13T01:30:54,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454195) roll requested 2024-11-13T01:30:54,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454202 2024-11-13T01:30:54,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454202 2024-11-13T01:30:54,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,209 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454202) roll requested 2024-11-13T01:30:54,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454210 2024-11-13T01:30:54,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454210 2024-11-13T01:30:54,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454210) roll requested 2024-11-13T01:30:54,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454220 2024-11-13T01:30:54,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454220 2024-11-13T01:30:54,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454220) roll requested 2024-11-13T01:30:54,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454230 2024-11-13T01:30:54,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454230 2024-11-13T01:30:54,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454230) roll requested 2024-11-13T01:30:54,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454240 2024-11-13T01:30:54,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454240 2024-11-13T01:30:54,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454240) roll requested 2024-11-13T01:30:54,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454250 2024-11-13T01:30:54,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454250 2024-11-13T01:30:54,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454250) roll requested 2024-11-13T01:30:54,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454259 2024-11-13T01:30:54,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454250 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454259 2024-11-13T01:30:54,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454259) roll requested 2024-11-13T01:30:54,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454267 2024-11-13T01:30:54,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454267 2024-11-13T01:30:54,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454267) roll requested 2024-11-13T01:30:54,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454276 2024-11-13T01:30:54,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454276 2024-11-13T01:30:54,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454276) roll requested 2024-11-13T01:30:54,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454285 2024-11-13T01:30:54,293 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454285 2024-11-13T01:30:54,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454285) roll requested 2024-11-13T01:30:54,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454295 2024-11-13T01:30:54,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454295 2024-11-13T01:30:54,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454295) roll requested 2024-11-13T01:30:54,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454305 2024-11-13T01:30:54,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454305 2024-11-13T01:30:54,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454305) roll requested 2024-11-13T01:30:54,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454313 2024-11-13T01:30:54,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454313 2024-11-13T01:30:54,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454313) roll requested 2024-11-13T01:30:54,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454320 2024-11-13T01:30:54,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454320 2024-11-13T01:30:54,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454320) roll requested 2024-11-13T01:30:54,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454328 2024-11-13T01:30:54,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454328 2024-11-13T01:30:54,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454328) roll requested 2024-11-13T01:30:54,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454333 2024-11-13T01:30:54,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454333 2024-11-13T01:30:54,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454333) roll requested 2024-11-13T01:30:54,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454338 2024-11-13T01:30:54,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454338 2024-11-13T01:30:54,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454338) roll requested 2024-11-13T01:30:54,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454345 2024-11-13T01:30:54,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454345 2024-11-13T01:30:54,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454345) roll requested 2024-11-13T01:30:54,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454352 2024-11-13T01:30:54,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454352 2024-11-13T01:30:54,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454352) roll requested 2024-11-13T01:30:54,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454359 2024-11-13T01:30:54,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454359 2024-11-13T01:30:54,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454359) roll requested 2024-11-13T01:30:54,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454365 2024-11-13T01:30:54,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454365 2024-11-13T01:30:54,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454365) roll requested 2024-11-13T01:30:54,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454371 2024-11-13T01:30:54,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454371 2024-11-13T01:30:54,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454371) roll requested 2024-11-13T01:30:54,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454437 2024-11-13T01:30:54,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454437 2024-11-13T01:30:54,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454437) roll requested 2024-11-13T01:30:54,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454444 2024-11-13T01:30:54,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454444 2024-11-13T01:30:54,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454444) roll requested 2024-11-13T01:30:54,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454453 2024-11-13T01:30:54,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454453 2024-11-13T01:30:54,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454453) roll requested 2024-11-13T01:30:54,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454459 2024-11-13T01:30:54,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454459 2024-11-13T01:30:54,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454459) roll requested 2024-11-13T01:30:54,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454464 2024-11-13T01:30:54,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454464 2024-11-13T01:30:54,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454464) roll requested 2024-11-13T01:30:54,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454470 2024-11-13T01:30:54,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454470 2024-11-13T01:30:54,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454470) roll requested 2024-11-13T01:30:54,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454478 2024-11-13T01:30:54,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454478 2024-11-13T01:30:54,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454478) roll requested 2024-11-13T01:30:54,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454486 2024-11-13T01:30:54,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454486 2024-11-13T01:30:54,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454486) roll requested 2024-11-13T01:30:54,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454495 2024-11-13T01:30:54,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454495 2024-11-13T01:30:54,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454495) roll requested 2024-11-13T01:30:54,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454503 2024-11-13T01:30:54,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454503 2024-11-13T01:30:54,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454503) roll requested 2024-11-13T01:30:54,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454509 2024-11-13T01:30:54,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454509 2024-11-13T01:30:54,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454509) roll requested 2024-11-13T01:30:54,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454516 2024-11-13T01:30:54,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454516 2024-11-13T01:30:54,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454516) roll requested 2024-11-13T01:30:54,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454524 2024-11-13T01:30:54,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454524 2024-11-13T01:30:54,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454524) roll requested 2024-11-13T01:30:54,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454532 2024-11-13T01:30:54,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454532 2024-11-13T01:30:54,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454532) roll requested 2024-11-13T01:30:54,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454540 2024-11-13T01:30:54,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454540 2024-11-13T01:30:54,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454540) roll requested 2024-11-13T01:30:54,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454550 2024-11-13T01:30:54,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454550 2024-11-13T01:30:54,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454550) roll requested 2024-11-13T01:30:54,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454561 2024-11-13T01:30:54,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454561 2024-11-13T01:30:54,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454561) roll requested 2024-11-13T01:30:54,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454569 2024-11-13T01:30:54,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454569 2024-11-13T01:30:54,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454569) roll requested 2024-11-13T01:30:54,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454575 2024-11-13T01:30:54,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454575 2024-11-13T01:30:54,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454575) roll requested 2024-11-13T01:30:54,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454583 2024-11-13T01:30:54,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454583 2024-11-13T01:30:54,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454583) roll requested 2024-11-13T01:30:54,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454592 2024-11-13T01:30:54,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454592 2024-11-13T01:30:54,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454592) roll requested 2024-11-13T01:30:54,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454602 2024-11-13T01:30:54,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454602 2024-11-13T01:30:54,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454602) roll requested 2024-11-13T01:30:54,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454608 2024-11-13T01:30:54,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454608 2024-11-13T01:30:54,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454608) roll requested 2024-11-13T01:30:54,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454620 2024-11-13T01:30:54,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454620 2024-11-13T01:30:54,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454620) roll requested 2024-11-13T01:30:54,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454628 2024-11-13T01:30:54,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454628 2024-11-13T01:30:54,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454628) roll requested 2024-11-13T01:30:54,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454640 2024-11-13T01:30:54,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454640 2024-11-13T01:30:54,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454640) roll requested 2024-11-13T01:30:54,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454648 2024-11-13T01:30:54,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454648 2024-11-13T01:30:54,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454648) roll requested 2024-11-13T01:30:54,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454656 2024-11-13T01:30:54,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454656 2024-11-13T01:30:54,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454656) roll requested 2024-11-13T01:30:54,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454664 2024-11-13T01:30:54,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454664 2024-11-13T01:30:54,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454664) roll requested 2024-11-13T01:30:54,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454672 2024-11-13T01:30:54,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454672 2024-11-13T01:30:54,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454672) roll requested 2024-11-13T01:30:54,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454678 2024-11-13T01:30:54,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454678 2024-11-13T01:30:54,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454678) roll requested 2024-11-13T01:30:54,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454683 2024-11-13T01:30:54,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454683 2024-11-13T01:30:54,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454683) roll requested 2024-11-13T01:30:54,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454689 2024-11-13T01:30:54,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454689 2024-11-13T01:30:54,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454689) roll requested 2024-11-13T01:30:54,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454695 2024-11-13T01:30:54,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454695 2024-11-13T01:30:54,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454695) roll requested 2024-11-13T01:30:54,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454700 2024-11-13T01:30:54,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454700 2024-11-13T01:30:54,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454700) roll requested 2024-11-13T01:30:54,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454706 2024-11-13T01:30:54,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454706 2024-11-13T01:30:54,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454706) roll requested 2024-11-13T01:30:54,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454711 2024-11-13T01:30:54,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454711 2024-11-13T01:30:54,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454711) roll requested 2024-11-13T01:30:54,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454716 2024-11-13T01:30:54,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454716 2024-11-13T01:30:54,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454716) roll requested 2024-11-13T01:30:54,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454722 2024-11-13T01:30:54,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454722 2024-11-13T01:30:54,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454722) roll requested 2024-11-13T01:30:54,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454728 2024-11-13T01:30:54,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454728 2024-11-13T01:30:54,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454728) roll requested 2024-11-13T01:30:54,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454734 2024-11-13T01:30:54,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454734 2024-11-13T01:30:54,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454734) roll requested 2024-11-13T01:30:54,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454742 2024-11-13T01:30:54,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454742 2024-11-13T01:30:54,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454742) roll requested 2024-11-13T01:30:54,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454748 2024-11-13T01:30:54,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454748 2024-11-13T01:30:54,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454748) roll requested 2024-11-13T01:30:54,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454753 2024-11-13T01:30:54,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454753 2024-11-13T01:30:54,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454753) roll requested 2024-11-13T01:30:54,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454759 2024-11-13T01:30:54,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454759 2024-11-13T01:30:54,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454759) roll requested 2024-11-13T01:30:54,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454766 2024-11-13T01:30:54,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454766 2024-11-13T01:30:54,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454766) roll requested 2024-11-13T01:30:54,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454771 2024-11-13T01:30:54,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454771 2024-11-13T01:30:54,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454771) roll requested 2024-11-13T01:30:54,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454776 2024-11-13T01:30:54,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454776 2024-11-13T01:30:54,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454776) roll requested 2024-11-13T01:30:54,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454783 2024-11-13T01:30:54,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454783 2024-11-13T01:30:54,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454783) roll requested 2024-11-13T01:30:54,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454788 2024-11-13T01:30:54,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454788 2024-11-13T01:30:54,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454788) roll requested 2024-11-13T01:30:54,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454794 2024-11-13T01:30:54,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454794 2024-11-13T01:30:54,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454794) roll requested 2024-11-13T01:30:54,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454799 2024-11-13T01:30:54,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454799 2024-11-13T01:30:54,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454799) roll requested 2024-11-13T01:30:54,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454805 2024-11-13T01:30:54,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454799 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454805 2024-11-13T01:30:54,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454805) roll requested 2024-11-13T01:30:54,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454815 2024-11-13T01:30:54,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454815 2024-11-13T01:30:54,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454815) roll requested 2024-11-13T01:30:54,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454822 2024-11-13T01:30:54,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454822 2024-11-13T01:30:54,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454822) roll requested 2024-11-13T01:30:54,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454831 2024-11-13T01:30:54,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454831 2024-11-13T01:30:54,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454831) roll requested 2024-11-13T01:30:54,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454836 2024-11-13T01:30:54,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454836 2024-11-13T01:30:54,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454836) roll requested 2024-11-13T01:30:54,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454842 2024-11-13T01:30:54,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454842 2024-11-13T01:30:54,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454842) roll requested 2024-11-13T01:30:54,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454848 2024-11-13T01:30:54,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454848 2024-11-13T01:30:54,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454848) roll requested 2024-11-13T01:30:54,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454855 2024-11-13T01:30:54,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454855 2024-11-13T01:30:54,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454855) roll requested 2024-11-13T01:30:54,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454863 2024-11-13T01:30:54,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454863 2024-11-13T01:30:54,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454863) roll requested 2024-11-13T01:30:54,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454871 2024-11-13T01:30:54,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454871 2024-11-13T01:30:54,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454871) roll requested 2024-11-13T01:30:54,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454879 2024-11-13T01:30:54,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454879 2024-11-13T01:30:54,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,886 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454879) roll requested 2024-11-13T01:30:54,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454887 2024-11-13T01:30:54,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454887 2024-11-13T01:30:54,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454887) roll requested 2024-11-13T01:30:54,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454895 2024-11-13T01:30:54,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454895 2024-11-13T01:30:54,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454895) roll requested 2024-11-13T01:30:54,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454903 2024-11-13T01:30:54,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454903 2024-11-13T01:30:54,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454903) roll requested 2024-11-13T01:30:54,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454911 2024-11-13T01:30:54,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454911 2024-11-13T01:30:54,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454911) roll requested 2024-11-13T01:30:54,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454920 2024-11-13T01:30:54,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454920 2024-11-13T01:30:54,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454920) roll requested 2024-11-13T01:30:54,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454929 2024-11-13T01:30:54,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454929 2024-11-13T01:30:54,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454929) roll requested 2024-11-13T01:30:54,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454936 2024-11-13T01:30:54,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454936 2024-11-13T01:30:54,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454936) roll requested 2024-11-13T01:30:54,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454943 2024-11-13T01:30:54,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454943 2024-11-13T01:30:54,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454943) roll requested 2024-11-13T01:30:54,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454951 2024-11-13T01:30:54,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454951 2024-11-13T01:30:54,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454951) roll requested 2024-11-13T01:30:54,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454959 2024-11-13T01:30:54,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454959 2024-11-13T01:30:54,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454959) roll requested 2024-11-13T01:30:54,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454968 2024-11-13T01:30:54,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454968 2024-11-13T01:30:54,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454968) roll requested 2024-11-13T01:30:54,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454980 2024-11-13T01:30:54,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454980 2024-11-13T01:30:54,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454980) roll requested 2024-11-13T01:30:54,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454987 2024-11-13T01:30:54,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454987 2024-11-13T01:30:54,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:54,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:54,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454987) roll requested 2024-11-13T01:30:54,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461454994 2024-11-13T01:30:54,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:54,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454994 2024-11-13T01:30:55,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461454994) roll requested 2024-11-13T01:30:55,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455000 2024-11-13T01:30:55,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461454994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455000 2024-11-13T01:30:55,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455000) roll requested 2024-11-13T01:30:55,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455007 2024-11-13T01:30:55,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455007 2024-11-13T01:30:55,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,012 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455007) roll requested 2024-11-13T01:30:55,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455013 2024-11-13T01:30:55,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455013 2024-11-13T01:30:55,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455013) roll requested 2024-11-13T01:30:55,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455018 2024-11-13T01:30:55,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455018 2024-11-13T01:30:55,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,024 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455018) roll requested 2024-11-13T01:30:55,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455024 2024-11-13T01:30:55,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455024 2024-11-13T01:30:55,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455024) roll requested 2024-11-13T01:30:55,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455031 2024-11-13T01:30:55,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455024 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455031 2024-11-13T01:30:55,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455031) roll requested 2024-11-13T01:30:55,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455039 2024-11-13T01:30:55,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455039 2024-11-13T01:30:55,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455039) roll requested 2024-11-13T01:30:55,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455049 2024-11-13T01:30:55,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455049 2024-11-13T01:30:55,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455049) roll requested 2024-11-13T01:30:55,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455058 2024-11-13T01:30:55,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455058 2024-11-13T01:30:55,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455058) roll requested 2024-11-13T01:30:55,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455065 2024-11-13T01:30:55,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455065 2024-11-13T01:30:55,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455065) roll requested 2024-11-13T01:30:55,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455074 2024-11-13T01:30:55,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455074 2024-11-13T01:30:55,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455074) roll requested 2024-11-13T01:30:55,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455083 2024-11-13T01:30:55,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455083 2024-11-13T01:30:55,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455083) roll requested 2024-11-13T01:30:55,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455090 2024-11-13T01:30:55,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455090 2024-11-13T01:30:55,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455090) roll requested 2024-11-13T01:30:55,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455098 2024-11-13T01:30:55,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455098 2024-11-13T01:30:55,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455098) roll requested 2024-11-13T01:30:55,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455106 2024-11-13T01:30:55,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455106 2024-11-13T01:30:55,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455106) roll requested 2024-11-13T01:30:55,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455120 2024-11-13T01:30:55,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455120 2024-11-13T01:30:55,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455120) roll requested 2024-11-13T01:30:55,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455130 2024-11-13T01:30:55,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455130 2024-11-13T01:30:55,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455130) roll requested 2024-11-13T01:30:55,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455136 2024-11-13T01:30:55,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455136 2024-11-13T01:30:55,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455136) roll requested 2024-11-13T01:30:55,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455142 2024-11-13T01:30:55,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455142 2024-11-13T01:30:55,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455142) roll requested 2024-11-13T01:30:55,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455150 2024-11-13T01:30:55,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455150 2024-11-13T01:30:55,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455150) roll requested 2024-11-13T01:30:55,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455158 2024-11-13T01:30:55,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455158 2024-11-13T01:30:55,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455158) roll requested 2024-11-13T01:30:55,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455167 2024-11-13T01:30:55,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455167 2024-11-13T01:30:55,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455167) roll requested 2024-11-13T01:30:55,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455176 2024-11-13T01:30:55,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455176 2024-11-13T01:30:55,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455176) roll requested 2024-11-13T01:30:55,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455183 2024-11-13T01:30:55,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455183 2024-11-13T01:30:55,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455183) roll requested 2024-11-13T01:30:55,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455189 2024-11-13T01:30:55,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455189 2024-11-13T01:30:55,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455189) roll requested 2024-11-13T01:30:55,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455196 2024-11-13T01:30:55,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455196 2024-11-13T01:30:55,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455196) roll requested 2024-11-13T01:30:55,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455203 2024-11-13T01:30:55,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455203 2024-11-13T01:30:55,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455203) roll requested 2024-11-13T01:30:55,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455211 2024-11-13T01:30:55,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455211 2024-11-13T01:30:55,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455211) roll requested 2024-11-13T01:30:55,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455220 2024-11-13T01:30:55,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455220 2024-11-13T01:30:55,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455220) roll requested 2024-11-13T01:30:55,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455230 2024-11-13T01:30:55,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455230 2024-11-13T01:30:55,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455230) roll requested 2024-11-13T01:30:55,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455240 2024-11-13T01:30:55,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455240 2024-11-13T01:30:55,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455240) roll requested 2024-11-13T01:30:55,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455248 2024-11-13T01:30:55,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455248 2024-11-13T01:30:55,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455248) roll requested 2024-11-13T01:30:55,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455256 2024-11-13T01:30:55,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455256 2024-11-13T01:30:55,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455256) roll requested 2024-11-13T01:30:55,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455265 2024-11-13T01:30:55,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455265 2024-11-13T01:30:55,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455265) roll requested 2024-11-13T01:30:55,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455273 2024-11-13T01:30:55,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455273 2024-11-13T01:30:55,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455273) roll requested 2024-11-13T01:30:55,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455283 2024-11-13T01:30:55,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455283 2024-11-13T01:30:55,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455283) roll requested 2024-11-13T01:30:55,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455291 2024-11-13T01:30:55,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455291 2024-11-13T01:30:55,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,303 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455291) roll requested 2024-11-13T01:30:55,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455303 2024-11-13T01:30:55,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455303 2024-11-13T01:30:55,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455303) roll requested 2024-11-13T01:30:55,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455308 2024-11-13T01:30:55,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455308 2024-11-13T01:30:55,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455308) roll requested 2024-11-13T01:30:55,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455316 2024-11-13T01:30:55,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455316 2024-11-13T01:30:55,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455316) roll requested 2024-11-13T01:30:55,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455324 2024-11-13T01:30:55,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455324 2024-11-13T01:30:55,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,332 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455324) roll requested 2024-11-13T01:30:55,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455332 2024-11-13T01:30:55,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455332 2024-11-13T01:30:55,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455332) roll requested 2024-11-13T01:30:55,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455340 2024-11-13T01:30:55,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455332 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455340 2024-11-13T01:30:55,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455340) roll requested 2024-11-13T01:30:55,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455349 2024-11-13T01:30:55,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455349 2024-11-13T01:30:55,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,354 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455349) roll requested 2024-11-13T01:30:55,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455355 2024-11-13T01:30:55,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455355 2024-11-13T01:30:55,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455355) roll requested 2024-11-13T01:30:55,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455363 2024-11-13T01:30:55,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455363 2024-11-13T01:30:55,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455363) roll requested 2024-11-13T01:30:55,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455371 2024-11-13T01:30:55,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455371 2024-11-13T01:30:55,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455371) roll requested 2024-11-13T01:30:55,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455376 2024-11-13T01:30:55,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455376 2024-11-13T01:30:55,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455376) roll requested 2024-11-13T01:30:55,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455382 2024-11-13T01:30:55,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455382 2024-11-13T01:30:55,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455382) roll requested 2024-11-13T01:30:55,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455388 2024-11-13T01:30:55,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455388 2024-11-13T01:30:55,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455388) roll requested 2024-11-13T01:30:55,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455394 2024-11-13T01:30:55,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455394 2024-11-13T01:30:55,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455394) roll requested 2024-11-13T01:30:55,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455400 2024-11-13T01:30:55,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455400 2024-11-13T01:30:55,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455400) roll requested 2024-11-13T01:30:55,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455404 2024-11-13T01:30:55,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455404 2024-11-13T01:30:55,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455404) roll requested 2024-11-13T01:30:55,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455415 2024-11-13T01:30:55,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455415 2024-11-13T01:30:55,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455415) roll requested 2024-11-13T01:30:55,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455421 2024-11-13T01:30:55,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,426 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455421 2024-11-13T01:30:55,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455421) roll requested 2024-11-13T01:30:55,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455427 2024-11-13T01:30:55,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455427 2024-11-13T01:30:55,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455427) roll requested 2024-11-13T01:30:55,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455433 2024-11-13T01:30:55,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455433 2024-11-13T01:30:55,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455433) roll requested 2024-11-13T01:30:55,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455438 2024-11-13T01:30:55,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455438 2024-11-13T01:30:55,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455438) roll requested 2024-11-13T01:30:55,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455443 2024-11-13T01:30:55,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455443 2024-11-13T01:30:55,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455443) roll requested 2024-11-13T01:30:55,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455450 2024-11-13T01:30:55,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455450 2024-11-13T01:30:55,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455450) roll requested 2024-11-13T01:30:55,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455457 2024-11-13T01:30:55,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455457 2024-11-13T01:30:55,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455457) roll requested 2024-11-13T01:30:55,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455465 2024-11-13T01:30:55,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455465 2024-11-13T01:30:55,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455465) roll requested 2024-11-13T01:30:55,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455475 2024-11-13T01:30:55,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455475 2024-11-13T01:30:55,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455475) roll requested 2024-11-13T01:30:55,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455481 2024-11-13T01:30:55,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455481 2024-11-13T01:30:55,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455481) roll requested 2024-11-13T01:30:55,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455487 2024-11-13T01:30:55,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455487 2024-11-13T01:30:55,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455487) roll requested 2024-11-13T01:30:55,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455492 2024-11-13T01:30:55,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455492 2024-11-13T01:30:55,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455492) roll requested 2024-11-13T01:30:55,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455497 2024-11-13T01:30:55,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455497 2024-11-13T01:30:55,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455497) roll requested 2024-11-13T01:30:55,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455503 2024-11-13T01:30:55,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455503 2024-11-13T01:30:55,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455503) roll requested 2024-11-13T01:30:55,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455508 2024-11-13T01:30:55,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455508 2024-11-13T01:30:55,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455508) roll requested 2024-11-13T01:30:55,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455516 2024-11-13T01:30:55,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455516 2024-11-13T01:30:55,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455516) roll requested 2024-11-13T01:30:55,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455521 2024-11-13T01:30:55,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455521 2024-11-13T01:30:55,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455521) roll requested 2024-11-13T01:30:55,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455529 2024-11-13T01:30:55,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455529 2024-11-13T01:30:55,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455529) roll requested 2024-11-13T01:30:55,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455535 2024-11-13T01:30:55,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455535 2024-11-13T01:30:55,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455535) roll requested 2024-11-13T01:30:55,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455540 2024-11-13T01:30:55,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455540 2024-11-13T01:30:55,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455540) roll requested 2024-11-13T01:30:55,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455546 2024-11-13T01:30:55,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455540 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455546 2024-11-13T01:30:55,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455546) roll requested 2024-11-13T01:30:55,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455552 2024-11-13T01:30:55,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455552 2024-11-13T01:30:55,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455552) roll requested 2024-11-13T01:30:55,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455557 2024-11-13T01:30:55,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455557 2024-11-13T01:30:55,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,561 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455557) roll requested 2024-11-13T01:30:55,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455562 2024-11-13T01:30:55,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455562 2024-11-13T01:30:55,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455562) roll requested 2024-11-13T01:30:55,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455570 2024-11-13T01:30:55,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455570 2024-11-13T01:30:55,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455570) roll requested 2024-11-13T01:30:55,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455576 2024-11-13T01:30:55,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455576 2024-11-13T01:30:55,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455576) roll requested 2024-11-13T01:30:55,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455582 2024-11-13T01:30:55,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455582 2024-11-13T01:30:55,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455582) roll requested 2024-11-13T01:30:55,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455588 2024-11-13T01:30:55,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455588 2024-11-13T01:30:55,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455588) roll requested 2024-11-13T01:30:55,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455593 2024-11-13T01:30:55,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455593 2024-11-13T01:30:55,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455593) roll requested 2024-11-13T01:30:55,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455599 2024-11-13T01:30:55,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455599 2024-11-13T01:30:55,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455599) roll requested 2024-11-13T01:30:55,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455604 2024-11-13T01:30:55,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455604 2024-11-13T01:30:55,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455604) roll requested 2024-11-13T01:30:55,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455610 2024-11-13T01:30:55,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455610 2024-11-13T01:30:55,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455610) roll requested 2024-11-13T01:30:55,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455617 2024-11-13T01:30:55,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455617 2024-11-13T01:30:55,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455617) roll requested 2024-11-13T01:30:55,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455625 2024-11-13T01:30:55,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455625 2024-11-13T01:30:55,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455625) roll requested 2024-11-13T01:30:55,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455632 2024-11-13T01:30:55,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455632 2024-11-13T01:30:55,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455632) roll requested 2024-11-13T01:30:55,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455638 2024-11-13T01:30:55,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455638 2024-11-13T01:30:55,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455638) roll requested 2024-11-13T01:30:55,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455643 2024-11-13T01:30:55,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455643 2024-11-13T01:30:55,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455643) roll requested 2024-11-13T01:30:55,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455649 2024-11-13T01:30:55,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455649 2024-11-13T01:30:55,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455649) roll requested 2024-11-13T01:30:55,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455655 2024-11-13T01:30:55,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455655 2024-11-13T01:30:55,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455655) roll requested 2024-11-13T01:30:55,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455663 2024-11-13T01:30:55,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455663 2024-11-13T01:30:55,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,671 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455663) roll requested 2024-11-13T01:30:55,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455672 2024-11-13T01:30:55,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455672 2024-11-13T01:30:55,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455672) roll requested 2024-11-13T01:30:55,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455682 2024-11-13T01:30:55,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455682 2024-11-13T01:30:55,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455682) roll requested 2024-11-13T01:30:55,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455688 2024-11-13T01:30:55,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455688 2024-11-13T01:30:55,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455688) roll requested 2024-11-13T01:30:55,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455695 2024-11-13T01:30:55,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455695 2024-11-13T01:30:55,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455695) roll requested 2024-11-13T01:30:55,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455703 2024-11-13T01:30:55,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455703 2024-11-13T01:30:55,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455703) roll requested 2024-11-13T01:30:55,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455712 2024-11-13T01:30:55,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455712 2024-11-13T01:30:55,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455712) roll requested 2024-11-13T01:30:55,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455718 2024-11-13T01:30:55,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455718 2024-11-13T01:30:55,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455718) roll requested 2024-11-13T01:30:55,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455726 2024-11-13T01:30:55,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455718 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455726 2024-11-13T01:30:55,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455726) roll requested 2024-11-13T01:30:55,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455736 2024-11-13T01:30:55,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455736 2024-11-13T01:30:55,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455736) roll requested 2024-11-13T01:30:55,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455745 2024-11-13T01:30:55,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455745 2024-11-13T01:30:55,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455745) roll requested 2024-11-13T01:30:55,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455752 2024-11-13T01:30:55,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455752 2024-11-13T01:30:55,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455752) roll requested 2024-11-13T01:30:55,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455760 2024-11-13T01:30:55,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455760 2024-11-13T01:30:55,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455760) roll requested 2024-11-13T01:30:55,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455767 2024-11-13T01:30:55,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455767 2024-11-13T01:30:55,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455767) roll requested 2024-11-13T01:30:55,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455775 2024-11-13T01:30:55,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455775 2024-11-13T01:30:55,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455775) roll requested 2024-11-13T01:30:55,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455784 2024-11-13T01:30:55,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455784 2024-11-13T01:30:55,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455784) roll requested 2024-11-13T01:30:55,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455794 2024-11-13T01:30:55,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455794 2024-11-13T01:30:55,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455794) roll requested 2024-11-13T01:30:55,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455801 2024-11-13T01:30:55,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455801 2024-11-13T01:30:55,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455801) roll requested 2024-11-13T01:30:55,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455808 2024-11-13T01:30:55,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455808 2024-11-13T01:30:55,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455808) roll requested 2024-11-13T01:30:55,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455816 2024-11-13T01:30:55,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455816 2024-11-13T01:30:55,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455816) roll requested 2024-11-13T01:30:55,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455825 2024-11-13T01:30:55,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455825 2024-11-13T01:30:55,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455825) roll requested 2024-11-13T01:30:55,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455836 2024-11-13T01:30:55,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455836 2024-11-13T01:30:55,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455836) roll requested 2024-11-13T01:30:55,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455844 2024-11-13T01:30:55,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455844 2024-11-13T01:30:55,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455844) roll requested 2024-11-13T01:30:55,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455852 2024-11-13T01:30:55,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455852 2024-11-13T01:30:55,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455852) roll requested 2024-11-13T01:30:55,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455859 2024-11-13T01:30:55,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455859 2024-11-13T01:30:55,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455859) roll requested 2024-11-13T01:30:55,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455866 2024-11-13T01:30:55,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455866 2024-11-13T01:30:55,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455866) roll requested 2024-11-13T01:30:55,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455876 2024-11-13T01:30:55,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455876 2024-11-13T01:30:55,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455876) roll requested 2024-11-13T01:30:55,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455884 2024-11-13T01:30:55,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455884 2024-11-13T01:30:55,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455884) roll requested 2024-11-13T01:30:55,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455892 2024-11-13T01:30:55,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455892 2024-11-13T01:30:55,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455892) roll requested 2024-11-13T01:30:55,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455899 2024-11-13T01:30:55,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455899 2024-11-13T01:30:55,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455899) roll requested 2024-11-13T01:30:55,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455907 2024-11-13T01:30:55,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455907 2024-11-13T01:30:55,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455907) roll requested 2024-11-13T01:30:55,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455914 2024-11-13T01:30:55,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455914 2024-11-13T01:30:55,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455914) roll requested 2024-11-13T01:30:55,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455921 2024-11-13T01:30:55,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455921 2024-11-13T01:30:55,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455921) roll requested 2024-11-13T01:30:55,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455929 2024-11-13T01:30:55,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455929 2024-11-13T01:30:55,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455929) roll requested 2024-11-13T01:30:55,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455936 2024-11-13T01:30:55,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455936 2024-11-13T01:30:55,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455936) roll requested 2024-11-13T01:30:55,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455944 2024-11-13T01:30:55,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455944 2024-11-13T01:30:55,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455944) roll requested 2024-11-13T01:30:55,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455951 2024-11-13T01:30:55,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455951 2024-11-13T01:30:55,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455951) roll requested 2024-11-13T01:30:55,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455960 2024-11-13T01:30:55,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455960 2024-11-13T01:30:55,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455960) roll requested 2024-11-13T01:30:55,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455971 2024-11-13T01:30:55,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455971 2024-11-13T01:30:55,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455971) roll requested 2024-11-13T01:30:55,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455979 2024-11-13T01:30:55,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455979 2024-11-13T01:30:55,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455979) roll requested 2024-11-13T01:30:55,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455988 2024-11-13T01:30:55,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:55,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455988 2024-11-13T01:30:55,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:55,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:55,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455988) roll requested 2024-11-13T01:30:55,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461455999 2024-11-13T01:30:56,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455999 2024-11-13T01:30:56,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461455999) roll requested 2024-11-13T01:30:56,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456004 2024-11-13T01:30:56,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461455999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456004 2024-11-13T01:30:56,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456004) roll requested 2024-11-13T01:30:56,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456009 2024-11-13T01:30:56,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456009 2024-11-13T01:30:56,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456009) roll requested 2024-11-13T01:30:56,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456017 2024-11-13T01:30:56,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456017 2024-11-13T01:30:56,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456017) roll requested 2024-11-13T01:30:56,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456025 2024-11-13T01:30:56,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456025 2024-11-13T01:30:56,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456025) roll requested 2024-11-13T01:30:56,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456031 2024-11-13T01:30:56,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456031 2024-11-13T01:30:56,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456031) roll requested 2024-11-13T01:30:56,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456040 2024-11-13T01:30:56,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456040 2024-11-13T01:30:56,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456040) roll requested 2024-11-13T01:30:56,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456046 2024-11-13T01:30:56,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456046 2024-11-13T01:30:56,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456046) roll requested 2024-11-13T01:30:56,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456052 2024-11-13T01:30:56,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456052 2024-11-13T01:30:56,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,057 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456052) roll requested 2024-11-13T01:30:56,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456057 2024-11-13T01:30:56,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456057 2024-11-13T01:30:56,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456057) roll requested 2024-11-13T01:30:56,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456062 2024-11-13T01:30:56,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456062 2024-11-13T01:30:56,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456062) roll requested 2024-11-13T01:30:56,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456070 2024-11-13T01:30:56,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456070 2024-11-13T01:30:56,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456070) roll requested 2024-11-13T01:30:56,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456077 2024-11-13T01:30:56,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456077 2024-11-13T01:30:56,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456077) roll requested 2024-11-13T01:30:56,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456085 2024-11-13T01:30:56,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456085 2024-11-13T01:30:56,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456085) roll requested 2024-11-13T01:30:56,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456093 2024-11-13T01:30:56,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456093 2024-11-13T01:30:56,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456093) roll requested 2024-11-13T01:30:56,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456102 2024-11-13T01:30:56,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456102 2024-11-13T01:30:56,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456102) roll requested 2024-11-13T01:30:56,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456111 2024-11-13T01:30:56,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456111 2024-11-13T01:30:56,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456111) roll requested 2024-11-13T01:30:56,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456120 2024-11-13T01:30:56,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456120 2024-11-13T01:30:56,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456120) roll requested 2024-11-13T01:30:56,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456129 2024-11-13T01:30:56,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456129 2024-11-13T01:30:56,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456129) roll requested 2024-11-13T01:30:56,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456138 2024-11-13T01:30:56,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456138 2024-11-13T01:30:56,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456138) roll requested 2024-11-13T01:30:56,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456146 2024-11-13T01:30:56,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456146 2024-11-13T01:30:56,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,156 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,156 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456146) roll requested 2024-11-13T01:30:56,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456156 2024-11-13T01:30:56,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456156 2024-11-13T01:30:56,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456156) roll requested 2024-11-13T01:30:56,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456161 2024-11-13T01:30:56,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456156 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456161 2024-11-13T01:30:56,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456161) roll requested 2024-11-13T01:30:56,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456166 2024-11-13T01:30:56,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456166 2024-11-13T01:30:56,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456166) roll requested 2024-11-13T01:30:56,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456171 2024-11-13T01:30:56,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456171 2024-11-13T01:30:56,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456171) roll requested 2024-11-13T01:30:56,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456177 2024-11-13T01:30:56,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456177 2024-11-13T01:30:56,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456177) roll requested 2024-11-13T01:30:56,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456184 2024-11-13T01:30:56,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456184 2024-11-13T01:30:56,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456184) roll requested 2024-11-13T01:30:56,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456193 2024-11-13T01:30:56,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456193 2024-11-13T01:30:56,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456193) roll requested 2024-11-13T01:30:56,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456202 2024-11-13T01:30:56,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456202 2024-11-13T01:30:56,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456202) roll requested 2024-11-13T01:30:56,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456209 2024-11-13T01:30:56,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456209 2024-11-13T01:30:56,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456209) roll requested 2024-11-13T01:30:56,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456216 2024-11-13T01:30:56,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456216 2024-11-13T01:30:56,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456216) roll requested 2024-11-13T01:30:56,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456222 2024-11-13T01:30:56,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456222 2024-11-13T01:30:56,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456222) roll requested 2024-11-13T01:30:56,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456228 2024-11-13T01:30:56,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456228 2024-11-13T01:30:56,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,234 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,234 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456228) roll requested 2024-11-13T01:30:56,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456234 2024-11-13T01:30:56,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456234 2024-11-13T01:30:56,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456234) roll requested 2024-11-13T01:30:56,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456242 2024-11-13T01:30:56,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456242 2024-11-13T01:30:56,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,249 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456242) roll requested 2024-11-13T01:30:56,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456249 2024-11-13T01:30:56,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456249 2024-11-13T01:30:56,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456249) roll requested 2024-11-13T01:30:56,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456256 2024-11-13T01:30:56,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456256 2024-11-13T01:30:56,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456256) roll requested 2024-11-13T01:30:56,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456262 2024-11-13T01:30:56,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456262 2024-11-13T01:30:56,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456262) roll requested 2024-11-13T01:30:56,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456269 2024-11-13T01:30:56,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456269 2024-11-13T01:30:56,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456269) roll requested 2024-11-13T01:30:56,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456277 2024-11-13T01:30:56,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456277 2024-11-13T01:30:56,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456277) roll requested 2024-11-13T01:30:56,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456284 2024-11-13T01:30:56,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456284 2024-11-13T01:30:56,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456284) roll requested 2024-11-13T01:30:56,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456293 2024-11-13T01:30:56,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456293 2024-11-13T01:30:56,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456293) roll requested 2024-11-13T01:30:56,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456299 2024-11-13T01:30:56,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456299 2024-11-13T01:30:56,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456299) roll requested 2024-11-13T01:30:56,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456308 2024-11-13T01:30:56,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456308 2024-11-13T01:30:56,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456308) roll requested 2024-11-13T01:30:56,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456316 2024-11-13T01:30:56,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456316 2024-11-13T01:30:56,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456316) roll requested 2024-11-13T01:30:56,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456323 2024-11-13T01:30:56,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456323 2024-11-13T01:30:56,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456323) roll requested 2024-11-13T01:30:56,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456329 2024-11-13T01:30:56,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456329 2024-11-13T01:30:56,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456329) roll requested 2024-11-13T01:30:56,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456334 2024-11-13T01:30:56,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456334 2024-11-13T01:30:56,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456334) roll requested 2024-11-13T01:30:56,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456340 2024-11-13T01:30:56,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456340 2024-11-13T01:30:56,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456340) roll requested 2024-11-13T01:30:56,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456347 2024-11-13T01:30:56,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456347 2024-11-13T01:30:56,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456347) roll requested 2024-11-13T01:30:56,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456359 2024-11-13T01:30:56,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456359 2024-11-13T01:30:56,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456359) roll requested 2024-11-13T01:30:56,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456364 2024-11-13T01:30:56,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456364 2024-11-13T01:30:56,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456364) roll requested 2024-11-13T01:30:56,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456370 2024-11-13T01:30:56,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456370 2024-11-13T01:30:56,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456370) roll requested 2024-11-13T01:30:56,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456376 2024-11-13T01:30:56,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456376 2024-11-13T01:30:56,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456376) roll requested 2024-11-13T01:30:56,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456383 2024-11-13T01:30:56,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456383 2024-11-13T01:30:56,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456383) roll requested 2024-11-13T01:30:56,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456388 2024-11-13T01:30:56,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456388 2024-11-13T01:30:56,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456388) roll requested 2024-11-13T01:30:56,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456396 2024-11-13T01:30:56,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456396 2024-11-13T01:30:56,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456396) roll requested 2024-11-13T01:30:56,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456402 2024-11-13T01:30:56,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456402 2024-11-13T01:30:56,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456402) roll requested 2024-11-13T01:30:56,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456408 2024-11-13T01:30:56,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456408 2024-11-13T01:30:56,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456408) roll requested 2024-11-13T01:30:56,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456415 2024-11-13T01:30:56,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456415 2024-11-13T01:30:56,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456415) roll requested 2024-11-13T01:30:56,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456420 2024-11-13T01:30:56,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456420 2024-11-13T01:30:56,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456420) roll requested 2024-11-13T01:30:56,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456425 2024-11-13T01:30:56,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456420 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456425 2024-11-13T01:30:56,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456425) roll requested 2024-11-13T01:30:56,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456431 2024-11-13T01:30:56,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456431 2024-11-13T01:30:56,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456431) roll requested 2024-11-13T01:30:56,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456437 2024-11-13T01:30:56,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456437 2024-11-13T01:30:56,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456437) roll requested 2024-11-13T01:30:56,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456442 2024-11-13T01:30:56,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456442 2024-11-13T01:30:56,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456442) roll requested 2024-11-13T01:30:56,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456449 2024-11-13T01:30:56,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456449 2024-11-13T01:30:56,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456449) roll requested 2024-11-13T01:30:56,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456456 2024-11-13T01:30:56,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456456 2024-11-13T01:30:56,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456456) roll requested 2024-11-13T01:30:56,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456462 2024-11-13T01:30:56,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456462 2024-11-13T01:30:56,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456462) roll requested 2024-11-13T01:30:56,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456467 2024-11-13T01:30:56,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456467 2024-11-13T01:30:56,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456467) roll requested 2024-11-13T01:30:56,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456473 2024-11-13T01:30:56,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456473 2024-11-13T01:30:56,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456473) roll requested 2024-11-13T01:30:56,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456479 2024-11-13T01:30:56,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456479 2024-11-13T01:30:56,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456479) roll requested 2024-11-13T01:30:56,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456484 2024-11-13T01:30:56,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456484 2024-11-13T01:30:56,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456484) roll requested 2024-11-13T01:30:56,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456489 2024-11-13T01:30:56,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456489 2024-11-13T01:30:56,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456489) roll requested 2024-11-13T01:30:56,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456496 2024-11-13T01:30:56,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456496 2024-11-13T01:30:56,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456496) roll requested 2024-11-13T01:30:56,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456502 2024-11-13T01:30:56,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456502 2024-11-13T01:30:56,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456502) roll requested 2024-11-13T01:30:56,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456510 2024-11-13T01:30:56,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456510 2024-11-13T01:30:56,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456510) roll requested 2024-11-13T01:30:56,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456518 2024-11-13T01:30:56,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456518 2024-11-13T01:30:56,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456518) roll requested 2024-11-13T01:30:56,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456526 2024-11-13T01:30:56,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456518 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456526 2024-11-13T01:30:56,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456526) roll requested 2024-11-13T01:30:56,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456532 2024-11-13T01:30:56,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456532 2024-11-13T01:30:56,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456532) roll requested 2024-11-13T01:30:56,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456539 2024-11-13T01:30:56,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456539 2024-11-13T01:30:56,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456539) roll requested 2024-11-13T01:30:56,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456545 2024-11-13T01:30:56,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456545 2024-11-13T01:30:56,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456545) roll requested 2024-11-13T01:30:56,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456552 2024-11-13T01:30:56,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456552 2024-11-13T01:30:56,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456552) roll requested 2024-11-13T01:30:56,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456560 2024-11-13T01:30:56,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456560 2024-11-13T01:30:56,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456560) roll requested 2024-11-13T01:30:56,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456567 2024-11-13T01:30:56,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456567 2024-11-13T01:30:56,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456567) roll requested 2024-11-13T01:30:56,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456574 2024-11-13T01:30:56,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456574 2024-11-13T01:30:56,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,582 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456574) roll requested 2024-11-13T01:30:56,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456583 2024-11-13T01:30:56,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456583 2024-11-13T01:30:56,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456583) roll requested 2024-11-13T01:30:56,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456591 2024-11-13T01:30:56,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456591 2024-11-13T01:30:56,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456591) roll requested 2024-11-13T01:30:56,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456599 2024-11-13T01:30:56,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456599 2024-11-13T01:30:56,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456599) roll requested 2024-11-13T01:30:56,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456606 2024-11-13T01:30:56,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456606 2024-11-13T01:30:56,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456606) roll requested 2024-11-13T01:30:56,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456612 2024-11-13T01:30:56,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456612 2024-11-13T01:30:56,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456612) roll requested 2024-11-13T01:30:56,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456618 2024-11-13T01:30:56,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456618 2024-11-13T01:30:56,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456618) roll requested 2024-11-13T01:30:56,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456626 2024-11-13T01:30:56,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456626 2024-11-13T01:30:56,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,634 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456626) roll requested 2024-11-13T01:30:56,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456635 2024-11-13T01:30:56,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456626 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456635 2024-11-13T01:30:56,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456635) roll requested 2024-11-13T01:30:56,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456644 2024-11-13T01:30:56,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456644 2024-11-13T01:30:56,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456644) roll requested 2024-11-13T01:30:56,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456652 2024-11-13T01:30:56,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456652 2024-11-13T01:30:56,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456652) roll requested 2024-11-13T01:30:56,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456663 2024-11-13T01:30:56,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456652 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456663 2024-11-13T01:30:56,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456663) roll requested 2024-11-13T01:30:56,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456670 2024-11-13T01:30:56,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456670 2024-11-13T01:30:56,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456670) roll requested 2024-11-13T01:30:56,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456679 2024-11-13T01:30:56,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456670 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456679 2024-11-13T01:30:56,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456679) roll requested 2024-11-13T01:30:56,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456688 2024-11-13T01:30:56,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456688 2024-11-13T01:30:56,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456688) roll requested 2024-11-13T01:30:56,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456697 2024-11-13T01:30:56,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456697 2024-11-13T01:30:56,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456697) roll requested 2024-11-13T01:30:56,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456705 2024-11-13T01:30:56,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456705 2024-11-13T01:30:56,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456705) roll requested 2024-11-13T01:30:56,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456711 2024-11-13T01:30:56,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456711 2024-11-13T01:30:56,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456711) roll requested 2024-11-13T01:30:56,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456720 2024-11-13T01:30:56,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456720 2024-11-13T01:30:56,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456720) roll requested 2024-11-13T01:30:56,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456728 2024-11-13T01:30:56,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456728 2024-11-13T01:30:56,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456728) roll requested 2024-11-13T01:30:56,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456736 2024-11-13T01:30:56,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456736 2024-11-13T01:30:56,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456736) roll requested 2024-11-13T01:30:56,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456742 2024-11-13T01:30:56,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456742 2024-11-13T01:30:56,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456742) roll requested 2024-11-13T01:30:56,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456748 2024-11-13T01:30:56,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456748 2024-11-13T01:30:56,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456748) roll requested 2024-11-13T01:30:56,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456755 2024-11-13T01:30:56,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456755 2024-11-13T01:30:56,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456755) roll requested 2024-11-13T01:30:56,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456762 2024-11-13T01:30:56,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456762 2024-11-13T01:30:56,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456762) roll requested 2024-11-13T01:30:56,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456769 2024-11-13T01:30:56,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456769 2024-11-13T01:30:56,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456769) roll requested 2024-11-13T01:30:56,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456778 2024-11-13T01:30:56,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456778 2024-11-13T01:30:56,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456778) roll requested 2024-11-13T01:30:56,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456786 2024-11-13T01:30:56,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456778 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456786 2024-11-13T01:30:56,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456786) roll requested 2024-11-13T01:30:56,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456793 2024-11-13T01:30:56,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456793 2024-11-13T01:30:56,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,798 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,798 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456793) roll requested 2024-11-13T01:30:56,798 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456798 2024-11-13T01:30:56,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,805 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,805 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456798 2024-11-13T01:30:56,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456798) roll requested 2024-11-13T01:30:56,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456806 2024-11-13T01:30:56,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456798 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456806 2024-11-13T01:30:56,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456806) roll requested 2024-11-13T01:30:56,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456814 2024-11-13T01:30:56,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456814 2024-11-13T01:30:56,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456814) roll requested 2024-11-13T01:30:56,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456823 2024-11-13T01:30:56,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456823 2024-11-13T01:30:56,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456823) roll requested 2024-11-13T01:30:56,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456830 2024-11-13T01:30:56,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456830 2024-11-13T01:30:56,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456830) roll requested 2024-11-13T01:30:56,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456837 2024-11-13T01:30:56,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456837 2024-11-13T01:30:56,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456837) roll requested 2024-11-13T01:30:56,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456847 2024-11-13T01:30:56,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456847 2024-11-13T01:30:56,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456847) roll requested 2024-11-13T01:30:56,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456856 2024-11-13T01:30:56,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456856 2024-11-13T01:30:56,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456856) roll requested 2024-11-13T01:30:56,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456864 2024-11-13T01:30:56,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456864 2024-11-13T01:30:56,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456864) roll requested 2024-11-13T01:30:56,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456874 2024-11-13T01:30:56,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456874 2024-11-13T01:30:56,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456874) roll requested 2024-11-13T01:30:56,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456881 2024-11-13T01:30:56,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456874 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456881 2024-11-13T01:30:56,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456881) roll requested 2024-11-13T01:30:56,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456889 2024-11-13T01:30:56,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456889 2024-11-13T01:30:56,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456889) roll requested 2024-11-13T01:30:56,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456898 2024-11-13T01:30:56,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456898 2024-11-13T01:30:56,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456898) roll requested 2024-11-13T01:30:56,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456908 2024-11-13T01:30:56,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456908 2024-11-13T01:30:56,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456908) roll requested 2024-11-13T01:30:56,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456918 2024-11-13T01:30:56,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456918 2024-11-13T01:30:56,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456918) roll requested 2024-11-13T01:30:56,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456927 2024-11-13T01:30:56,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456927 2024-11-13T01:30:56,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456927) roll requested 2024-11-13T01:30:56,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456935 2024-11-13T01:30:56,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456935 2024-11-13T01:30:56,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456935) roll requested 2024-11-13T01:30:56,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456944 2024-11-13T01:30:56,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456944 2024-11-13T01:30:56,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456944) roll requested 2024-11-13T01:30:56,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456952 2024-11-13T01:30:56,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456952 2024-11-13T01:30:56,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456952) roll requested 2024-11-13T01:30:56,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456962 2024-11-13T01:30:56,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456962 2024-11-13T01:30:56,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456962) roll requested 2024-11-13T01:30:56,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456971 2024-11-13T01:30:56,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456971 2024-11-13T01:30:56,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456971) roll requested 2024-11-13T01:30:56,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456977 2024-11-13T01:30:56,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456971 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456977 2024-11-13T01:30:56,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456977) roll requested 2024-11-13T01:30:56,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456983 2024-11-13T01:30:56,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456983 2024-11-13T01:30:56,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,990 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456983) roll requested 2024-11-13T01:30:56,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456991 2024-11-13T01:30:56,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:56,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456991 2024-11-13T01:30:56,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:56,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:56,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456991) roll requested 2024-11-13T01:30:56,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461456999 2024-11-13T01:30:57,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456999 2024-11-13T01:30:57,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461456999) roll requested 2024-11-13T01:30:57,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457007 2024-11-13T01:30:57,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461456999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457007 2024-11-13T01:30:57,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457007) roll requested 2024-11-13T01:30:57,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457016 2024-11-13T01:30:57,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457016 2024-11-13T01:30:57,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457016) roll requested 2024-11-13T01:30:57,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457022 2024-11-13T01:30:57,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457022 2024-11-13T01:30:57,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457022) roll requested 2024-11-13T01:30:57,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457030 2024-11-13T01:30:57,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457030 2024-11-13T01:30:57,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457030) roll requested 2024-11-13T01:30:57,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457039 2024-11-13T01:30:57,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457039 2024-11-13T01:30:57,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457039) roll requested 2024-11-13T01:30:57,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457048 2024-11-13T01:30:57,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457048 2024-11-13T01:30:57,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457048) roll requested 2024-11-13T01:30:57,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457058 2024-11-13T01:30:57,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457058 2024-11-13T01:30:57,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457058) roll requested 2024-11-13T01:30:57,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457067 2024-11-13T01:30:57,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457067 2024-11-13T01:30:57,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,075 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457067) roll requested 2024-11-13T01:30:57,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457076 2024-11-13T01:30:57,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457076 2024-11-13T01:30:57,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457076) roll requested 2024-11-13T01:30:57,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457085 2024-11-13T01:30:57,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457085 2024-11-13T01:30:57,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457085) roll requested 2024-11-13T01:30:57,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457091 2024-11-13T01:30:57,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457091 2024-11-13T01:30:57,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457091) roll requested 2024-11-13T01:30:57,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457098 2024-11-13T01:30:57,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457098 2024-11-13T01:30:57,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457098) roll requested 2024-11-13T01:30:57,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457106 2024-11-13T01:30:57,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457106 2024-11-13T01:30:57,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457106) roll requested 2024-11-13T01:30:57,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457113 2024-11-13T01:30:57,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457113 2024-11-13T01:30:57,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457113) roll requested 2024-11-13T01:30:57,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457123 2024-11-13T01:30:57,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457123 2024-11-13T01:30:57,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457123) roll requested 2024-11-13T01:30:57,131 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457131 2024-11-13T01:30:57,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457131 2024-11-13T01:30:57,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457131) roll requested 2024-11-13T01:30:57,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457140 2024-11-13T01:30:57,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457140 2024-11-13T01:30:57,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457140) roll requested 2024-11-13T01:30:57,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457151 2024-11-13T01:30:57,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457151 2024-11-13T01:30:57,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457151) roll requested 2024-11-13T01:30:57,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457160 2024-11-13T01:30:57,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457160 2024-11-13T01:30:57,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,169 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457160) roll requested 2024-11-13T01:30:57,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457169 2024-11-13T01:30:57,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457169 2024-11-13T01:30:57,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457169) roll requested 2024-11-13T01:30:57,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457176 2024-11-13T01:30:57,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457169 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457176 2024-11-13T01:30:57,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457176) roll requested 2024-11-13T01:30:57,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457182 2024-11-13T01:30:57,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457182 2024-11-13T01:30:57,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457182) roll requested 2024-11-13T01:30:57,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457190 2024-11-13T01:30:57,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457190 2024-11-13T01:30:57,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457190) roll requested 2024-11-13T01:30:57,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457199 2024-11-13T01:30:57,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457199 2024-11-13T01:30:57,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457199) roll requested 2024-11-13T01:30:57,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457207 2024-11-13T01:30:57,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457207 2024-11-13T01:30:57,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,217 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457207) roll requested 2024-11-13T01:30:57,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457217 2024-11-13T01:30:57,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457217 2024-11-13T01:30:57,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457217) roll requested 2024-11-13T01:30:57,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457225 2024-11-13T01:30:57,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457225 2024-11-13T01:30:57,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457225) roll requested 2024-11-13T01:30:57,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457231 2024-11-13T01:30:57,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457231 2024-11-13T01:30:57,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457231) roll requested 2024-11-13T01:30:57,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457239 2024-11-13T01:30:57,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457239 2024-11-13T01:30:57,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457239) roll requested 2024-11-13T01:30:57,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457248 2024-11-13T01:30:57,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457248 2024-11-13T01:30:57,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457248) roll requested 2024-11-13T01:30:57,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457255 2024-11-13T01:30:57,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457255 2024-11-13T01:30:57,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457255) roll requested 2024-11-13T01:30:57,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457261 2024-11-13T01:30:57,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457261 2024-11-13T01:30:57,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457261) roll requested 2024-11-13T01:30:57,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457268 2024-11-13T01:30:57,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457268 2024-11-13T01:30:57,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457268) roll requested 2024-11-13T01:30:57,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457275 2024-11-13T01:30:57,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457275 2024-11-13T01:30:57,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457275) roll requested 2024-11-13T01:30:57,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457283 2024-11-13T01:30:57,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457283 2024-11-13T01:30:57,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457283) roll requested 2024-11-13T01:30:57,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457292 2024-11-13T01:30:57,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457292 2024-11-13T01:30:57,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457292) roll requested 2024-11-13T01:30:57,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457301 2024-11-13T01:30:57,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457301 2024-11-13T01:30:57,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457301) roll requested 2024-11-13T01:30:57,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457311 2024-11-13T01:30:57,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457311 2024-11-13T01:30:57,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457311) roll requested 2024-11-13T01:30:57,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457320 2024-11-13T01:30:57,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457320 2024-11-13T01:30:57,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457320) roll requested 2024-11-13T01:30:57,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457331 2024-11-13T01:30:57,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457331 2024-11-13T01:30:57,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457331) roll requested 2024-11-13T01:30:57,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457339 2024-11-13T01:30:57,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457339 2024-11-13T01:30:57,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457339) roll requested 2024-11-13T01:30:57,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457344 2024-11-13T01:30:57,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457344 2024-11-13T01:30:57,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457344) roll requested 2024-11-13T01:30:57,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457349 2024-11-13T01:30:57,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457349 2024-11-13T01:30:57,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457349) roll requested 2024-11-13T01:30:57,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457356 2024-11-13T01:30:57,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457356 2024-11-13T01:30:57,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457356) roll requested 2024-11-13T01:30:57,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457363 2024-11-13T01:30:57,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457363 2024-11-13T01:30:57,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457363) roll requested 2024-11-13T01:30:57,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457371 2024-11-13T01:30:57,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457371 2024-11-13T01:30:57,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457371) roll requested 2024-11-13T01:30:57,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457379 2024-11-13T01:30:57,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457379 2024-11-13T01:30:57,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457379) roll requested 2024-11-13T01:30:57,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457388 2024-11-13T01:30:57,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457388 2024-11-13T01:30:57,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457388) roll requested 2024-11-13T01:30:57,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457394 2024-11-13T01:30:57,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457394 2024-11-13T01:30:57,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457394) roll requested 2024-11-13T01:30:57,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457399 2024-11-13T01:30:57,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457399 2024-11-13T01:30:57,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457399) roll requested 2024-11-13T01:30:57,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457407 2024-11-13T01:30:57,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457407 2024-11-13T01:30:57,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457407) roll requested 2024-11-13T01:30:57,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457414 2024-11-13T01:30:57,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457414 2024-11-13T01:30:57,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457414) roll requested 2024-11-13T01:30:57,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457424 2024-11-13T01:30:57,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457424 2024-11-13T01:30:57,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457424) roll requested 2024-11-13T01:30:57,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457431 2024-11-13T01:30:57,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457431 2024-11-13T01:30:57,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457431) roll requested 2024-11-13T01:30:57,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457440 2024-11-13T01:30:57,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457440 2024-11-13T01:30:57,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457440) roll requested 2024-11-13T01:30:57,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457450 2024-11-13T01:30:57,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,455 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457450 2024-11-13T01:30:57,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457450) roll requested 2024-11-13T01:30:57,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457456 2024-11-13T01:30:57,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457456 2024-11-13T01:30:57,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457456) roll requested 2024-11-13T01:30:57,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457462 2024-11-13T01:30:57,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457462 2024-11-13T01:30:57,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,470 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457462) roll requested 2024-11-13T01:30:57,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457470 2024-11-13T01:30:57,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457470 2024-11-13T01:30:57,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457470) roll requested 2024-11-13T01:30:57,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457479 2024-11-13T01:30:57,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457479 2024-11-13T01:30:57,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457479) roll requested 2024-11-13T01:30:57,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457488 2024-11-13T01:30:57,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457488 2024-11-13T01:30:57,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457488) roll requested 2024-11-13T01:30:57,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457497 2024-11-13T01:30:57,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457497 2024-11-13T01:30:57,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457497) roll requested 2024-11-13T01:30:57,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457502 2024-11-13T01:30:57,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457502 2024-11-13T01:30:57,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457502) roll requested 2024-11-13T01:30:57,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457507 2024-11-13T01:30:57,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457507 2024-11-13T01:30:57,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,513 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457507) roll requested 2024-11-13T01:30:57,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457513 2024-11-13T01:30:57,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457513 2024-11-13T01:30:57,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457513) roll requested 2024-11-13T01:30:57,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457519 2024-11-13T01:30:57,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457519 2024-11-13T01:30:57,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457519) roll requested 2024-11-13T01:30:57,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457528 2024-11-13T01:30:57,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457528 2024-11-13T01:30:57,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457528) roll requested 2024-11-13T01:30:57,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457535 2024-11-13T01:30:57,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457535 2024-11-13T01:30:57,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457535) roll requested 2024-11-13T01:30:57,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457543 2024-11-13T01:30:57,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457543 2024-11-13T01:30:57,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457543) roll requested 2024-11-13T01:30:57,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457552 2024-11-13T01:30:57,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457552 2024-11-13T01:30:57,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457552) roll requested 2024-11-13T01:30:57,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457557 2024-11-13T01:30:57,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457557 2024-11-13T01:30:57,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457557) roll requested 2024-11-13T01:30:57,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457562 2024-11-13T01:30:57,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457562 2024-11-13T01:30:57,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457562) roll requested 2024-11-13T01:30:57,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457568 2024-11-13T01:30:57,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457568 2024-11-13T01:30:57,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457568) roll requested 2024-11-13T01:30:57,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457576 2024-11-13T01:30:57,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457576 2024-11-13T01:30:57,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,584 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457576) roll requested 2024-11-13T01:30:57,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457584 2024-11-13T01:30:57,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457584 2024-11-13T01:30:57,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457584) roll requested 2024-11-13T01:30:57,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457592 2024-11-13T01:30:57,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457592 2024-11-13T01:30:57,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457592) roll requested 2024-11-13T01:30:57,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457597 2024-11-13T01:30:57,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457597 2024-11-13T01:30:57,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457597) roll requested 2024-11-13T01:30:57,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457606 2024-11-13T01:30:57,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457606 2024-11-13T01:30:57,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457606) roll requested 2024-11-13T01:30:57,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457612 2024-11-13T01:30:57,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457612 2024-11-13T01:30:57,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457612) roll requested 2024-11-13T01:30:57,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457618 2024-11-13T01:30:57,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457618 2024-11-13T01:30:57,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457618) roll requested 2024-11-13T01:30:57,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457623 2024-11-13T01:30:57,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457623 2024-11-13T01:30:57,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457623) roll requested 2024-11-13T01:30:57,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457629 2024-11-13T01:30:57,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457629 2024-11-13T01:30:57,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457629) roll requested 2024-11-13T01:30:57,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457636 2024-11-13T01:30:57,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457636 2024-11-13T01:30:57,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457636) roll requested 2024-11-13T01:30:57,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457644 2024-11-13T01:30:57,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457644 2024-11-13T01:30:57,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457644) roll requested 2024-11-13T01:30:57,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457651 2024-11-13T01:30:57,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457651 2024-11-13T01:30:57,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457651) roll requested 2024-11-13T01:30:57,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457661 2024-11-13T01:30:57,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457661 2024-11-13T01:30:57,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,669 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457661) roll requested 2024-11-13T01:30:57,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457669 2024-11-13T01:30:57,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457661 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457669 2024-11-13T01:30:57,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457669) roll requested 2024-11-13T01:30:57,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457676 2024-11-13T01:30:57,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457676 2024-11-13T01:30:57,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457676) roll requested 2024-11-13T01:30:57,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457684 2024-11-13T01:30:57,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457684 2024-11-13T01:30:57,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457684) roll requested 2024-11-13T01:30:57,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457692 2024-11-13T01:30:57,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457692 2024-11-13T01:30:57,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457692) roll requested 2024-11-13T01:30:57,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457701 2024-11-13T01:30:57,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457701 2024-11-13T01:30:57,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457701) roll requested 2024-11-13T01:30:57,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457709 2024-11-13T01:30:57,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457709 2024-11-13T01:30:57,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457709) roll requested 2024-11-13T01:30:57,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457714 2024-11-13T01:30:57,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457714 2024-11-13T01:30:57,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457714) roll requested 2024-11-13T01:30:57,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457725 2024-11-13T01:30:57,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457725 2024-11-13T01:30:57,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457725) roll requested 2024-11-13T01:30:57,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457735 2024-11-13T01:30:57,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457735 2024-11-13T01:30:57,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457735) roll requested 2024-11-13T01:30:57,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457741 2024-11-13T01:30:57,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457741 2024-11-13T01:30:57,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457741) roll requested 2024-11-13T01:30:57,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457747 2024-11-13T01:30:57,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457747 2024-11-13T01:30:57,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457747) roll requested 2024-11-13T01:30:57,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457754 2024-11-13T01:30:57,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457754 2024-11-13T01:30:57,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457754) roll requested 2024-11-13T01:30:57,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457760 2024-11-13T01:30:57,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457760 2024-11-13T01:30:57,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457760) roll requested 2024-11-13T01:30:57,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457768 2024-11-13T01:30:57,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457768 2024-11-13T01:30:57,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457768) roll requested 2024-11-13T01:30:57,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457775 2024-11-13T01:30:57,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457775 2024-11-13T01:30:57,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457775) roll requested 2024-11-13T01:30:57,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457784 2024-11-13T01:30:57,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457784 2024-11-13T01:30:57,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,790 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457784) roll requested 2024-11-13T01:30:57,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457791 2024-11-13T01:30:57,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457791 2024-11-13T01:30:57,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457791) roll requested 2024-11-13T01:30:57,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457796 2024-11-13T01:30:57,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457796 2024-11-13T01:30:57,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457796) roll requested 2024-11-13T01:30:57,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457802 2024-11-13T01:30:57,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457802 2024-11-13T01:30:57,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457802) roll requested 2024-11-13T01:30:57,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457808 2024-11-13T01:30:57,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457808 2024-11-13T01:30:57,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457808) roll requested 2024-11-13T01:30:57,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457816 2024-11-13T01:30:57,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457816 2024-11-13T01:30:57,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457816) roll requested 2024-11-13T01:30:57,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457823 2024-11-13T01:30:57,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457823 2024-11-13T01:30:57,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457823) roll requested 2024-11-13T01:30:57,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457829 2024-11-13T01:30:57,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457829 2024-11-13T01:30:57,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457829) roll requested 2024-11-13T01:30:57,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457835 2024-11-13T01:30:57,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457835 2024-11-13T01:30:57,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457835) roll requested 2024-11-13T01:30:57,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457842 2024-11-13T01:30:57,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457842 2024-11-13T01:30:57,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457842) roll requested 2024-11-13T01:30:57,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457850 2024-11-13T01:30:57,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457850 2024-11-13T01:30:57,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457850) roll requested 2024-11-13T01:30:57,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457856 2024-11-13T01:30:57,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457856 2024-11-13T01:30:57,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457856) roll requested 2024-11-13T01:30:57,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457864 2024-11-13T01:30:57,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457864 2024-11-13T01:30:57,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457864) roll requested 2024-11-13T01:30:57,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457872 2024-11-13T01:30:57,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457872 2024-11-13T01:30:57,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457872) roll requested 2024-11-13T01:30:57,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457880 2024-11-13T01:30:57,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457880 2024-11-13T01:30:57,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457880) roll requested 2024-11-13T01:30:57,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457885 2024-11-13T01:30:57,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457885 2024-11-13T01:30:57,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457885) roll requested 2024-11-13T01:30:57,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457890 2024-11-13T01:30:57,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457890 2024-11-13T01:30:57,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457890) roll requested 2024-11-13T01:30:57,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457896 2024-11-13T01:30:57,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457896 2024-11-13T01:30:57,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457896) roll requested 2024-11-13T01:30:57,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457902 2024-11-13T01:30:57,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457902 2024-11-13T01:30:57,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457902) roll requested 2024-11-13T01:30:57,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457908 2024-11-13T01:30:57,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457908 2024-11-13T01:30:57,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457908) roll requested 2024-11-13T01:30:57,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457915 2024-11-13T01:30:57,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457915 2024-11-13T01:30:57,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457915) roll requested 2024-11-13T01:30:57,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457924 2024-11-13T01:30:57,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457924 2024-11-13T01:30:57,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457924) roll requested 2024-11-13T01:30:57,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457934 2024-11-13T01:30:57,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457934 2024-11-13T01:30:57,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457934) roll requested 2024-11-13T01:30:57,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457940 2024-11-13T01:30:57,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457940 2024-11-13T01:30:57,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457940) roll requested 2024-11-13T01:30:57,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457948 2024-11-13T01:30:57,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457948 2024-11-13T01:30:57,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457948) roll requested 2024-11-13T01:30:57,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457956 2024-11-13T01:30:57,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457956 2024-11-13T01:30:57,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457956) roll requested 2024-11-13T01:30:57,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457964 2024-11-13T01:30:57,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457964 2024-11-13T01:30:57,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457964) roll requested 2024-11-13T01:30:57,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457973 2024-11-13T01:30:57,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457973 2024-11-13T01:30:57,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457973) roll requested 2024-11-13T01:30:57,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457982 2024-11-13T01:30:57,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457982 2024-11-13T01:30:57,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457982) roll requested 2024-11-13T01:30:57,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461457993 2024-11-13T01:30:57,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:57,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457993 2024-11-13T01:30:57,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:57,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:57,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461457993) roll requested 2024-11-13T01:30:58,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458000 2024-11-13T01:30:58,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461457993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458000 2024-11-13T01:30:58,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458000) roll requested 2024-11-13T01:30:58,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458006 2024-11-13T01:30:58,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458006 2024-11-13T01:30:58,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458006) roll requested 2024-11-13T01:30:58,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458015 2024-11-13T01:30:58,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458015 2024-11-13T01:30:58,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458015) roll requested 2024-11-13T01:30:58,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458023 2024-11-13T01:30:58,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458023 2024-11-13T01:30:58,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,032 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458023) roll requested 2024-11-13T01:30:58,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458033 2024-11-13T01:30:58,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458033 2024-11-13T01:30:58,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458033) roll requested 2024-11-13T01:30:58,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458039 2024-11-13T01:30:58,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458039 2024-11-13T01:30:58,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458039) roll requested 2024-11-13T01:30:58,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458047 2024-11-13T01:30:58,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458047 2024-11-13T01:30:58,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458047) roll requested 2024-11-13T01:30:58,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458055 2024-11-13T01:30:58,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458055 2024-11-13T01:30:58,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458055) roll requested 2024-11-13T01:30:58,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458064 2024-11-13T01:30:58,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458064 2024-11-13T01:30:58,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458064) roll requested 2024-11-13T01:30:58,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458073 2024-11-13T01:30:58,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458073 2024-11-13T01:30:58,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458073) roll requested 2024-11-13T01:30:58,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458078 2024-11-13T01:30:58,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458078 2024-11-13T01:30:58,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458078) roll requested 2024-11-13T01:30:58,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458084 2024-11-13T01:30:58,090 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,090 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458084 2024-11-13T01:30:58,091 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,091 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458084) roll requested 2024-11-13T01:30:58,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458091 2024-11-13T01:30:58,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458091 2024-11-13T01:30:58,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458091) roll requested 2024-11-13T01:30:58,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458099 2024-11-13T01:30:58,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458091 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458099 2024-11-13T01:30:58,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458099) roll requested 2024-11-13T01:30:58,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458106 2024-11-13T01:30:58,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458106 2024-11-13T01:30:58,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458106) roll requested 2024-11-13T01:30:58,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458113 2024-11-13T01:30:58,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458113 2024-11-13T01:30:58,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458113) roll requested 2024-11-13T01:30:58,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458121 2024-11-13T01:30:58,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458121 2024-11-13T01:30:58,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458121) roll requested 2024-11-13T01:30:58,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458130 2024-11-13T01:30:58,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458130 2024-11-13T01:30:58,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458130) roll requested 2024-11-13T01:30:58,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458139 2024-11-13T01:30:58,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458139 2024-11-13T01:30:58,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458139) roll requested 2024-11-13T01:30:58,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458149 2024-11-13T01:30:58,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458149 2024-11-13T01:30:58,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458149) roll requested 2024-11-13T01:30:58,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458155 2024-11-13T01:30:58,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458155 2024-11-13T01:30:58,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458155) roll requested 2024-11-13T01:30:58,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458165 2024-11-13T01:30:58,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458165 2024-11-13T01:30:58,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458165) roll requested 2024-11-13T01:30:58,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458172 2024-11-13T01:30:58,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458172 2024-11-13T01:30:58,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,177 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458172) roll requested 2024-11-13T01:30:58,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458177 2024-11-13T01:30:58,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458177 2024-11-13T01:30:58,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458177) roll requested 2024-11-13T01:30:58,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458183 2024-11-13T01:30:58,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458183 2024-11-13T01:30:58,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458183) roll requested 2024-11-13T01:30:58,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458188 2024-11-13T01:30:58,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458188 2024-11-13T01:30:58,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458188) roll requested 2024-11-13T01:30:58,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458194 2024-11-13T01:30:58,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458194 2024-11-13T01:30:58,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458194) roll requested 2024-11-13T01:30:58,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458199 2024-11-13T01:30:58,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458199 2024-11-13T01:30:58,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458199) roll requested 2024-11-13T01:30:58,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458204 2024-11-13T01:30:58,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458204 2024-11-13T01:30:58,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458204) roll requested 2024-11-13T01:30:58,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458214 2024-11-13T01:30:58,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458214 2024-11-13T01:30:58,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458214) roll requested 2024-11-13T01:30:58,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458222 2024-11-13T01:30:58,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458222 2024-11-13T01:30:58,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458222) roll requested 2024-11-13T01:30:58,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458229 2024-11-13T01:30:58,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458229 2024-11-13T01:30:58,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458229) roll requested 2024-11-13T01:30:58,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458238 2024-11-13T01:30:58,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458238 2024-11-13T01:30:58,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458238) roll requested 2024-11-13T01:30:58,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458246 2024-11-13T01:30:58,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458246 2024-11-13T01:30:58,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458246) roll requested 2024-11-13T01:30:58,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458252 2024-11-13T01:30:58,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458252 2024-11-13T01:30:58,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458252) roll requested 2024-11-13T01:30:58,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458260 2024-11-13T01:30:58,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458260 2024-11-13T01:30:58,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458260) roll requested 2024-11-13T01:30:58,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458268 2024-11-13T01:30:58,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458268 2024-11-13T01:30:58,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458268) roll requested 2024-11-13T01:30:58,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458275 2024-11-13T01:30:58,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458275 2024-11-13T01:30:58,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458275) roll requested 2024-11-13T01:30:58,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458283 2024-11-13T01:30:58,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458283 2024-11-13T01:30:58,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458283) roll requested 2024-11-13T01:30:58,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458292 2024-11-13T01:30:58,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458292 2024-11-13T01:30:58,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458292) roll requested 2024-11-13T01:30:58,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458301 2024-11-13T01:30:58,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458301 2024-11-13T01:30:58,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458301) roll requested 2024-11-13T01:30:58,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458307 2024-11-13T01:30:58,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458307 2024-11-13T01:30:58,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458307) roll requested 2024-11-13T01:30:58,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458314 2024-11-13T01:30:58,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458314 2024-11-13T01:30:58,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458314) roll requested 2024-11-13T01:30:58,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458323 2024-11-13T01:30:58,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458323 2024-11-13T01:30:58,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458323) roll requested 2024-11-13T01:30:58,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458331 2024-11-13T01:30:58,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458331 2024-11-13T01:30:58,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458331) roll requested 2024-11-13T01:30:58,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458340 2024-11-13T01:30:58,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458340 2024-11-13T01:30:58,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458340) roll requested 2024-11-13T01:30:58,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458350 2024-11-13T01:30:58,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458350 2024-11-13T01:30:58,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458350) roll requested 2024-11-13T01:30:58,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458358 2024-11-13T01:30:58,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458358 2024-11-13T01:30:58,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458358) roll requested 2024-11-13T01:30:58,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458367 2024-11-13T01:30:58,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458367 2024-11-13T01:30:58,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458367) roll requested 2024-11-13T01:30:58,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458377 2024-11-13T01:30:58,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458377 2024-11-13T01:30:58,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458377) roll requested 2024-11-13T01:30:58,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458387 2024-11-13T01:30:58,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458387 2024-11-13T01:30:58,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458387) roll requested 2024-11-13T01:30:58,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458396 2024-11-13T01:30:58,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458396 2024-11-13T01:30:58,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458396) roll requested 2024-11-13T01:30:58,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458404 2024-11-13T01:30:58,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458404 2024-11-13T01:30:58,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458404) roll requested 2024-11-13T01:30:58,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458415 2024-11-13T01:30:58,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458415 2024-11-13T01:30:58,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458415) roll requested 2024-11-13T01:30:58,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458425 2024-11-13T01:30:58,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458425 2024-11-13T01:30:58,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458425) roll requested 2024-11-13T01:30:58,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458433 2024-11-13T01:30:58,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458433 2024-11-13T01:30:58,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,441 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458433) roll requested 2024-11-13T01:30:58,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458441 2024-11-13T01:30:58,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458441 2024-11-13T01:30:58,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458441) roll requested 2024-11-13T01:30:58,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458451 2024-11-13T01:30:58,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458451 2024-11-13T01:30:58,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458451) roll requested 2024-11-13T01:30:58,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458458 2024-11-13T01:30:58,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458458 2024-11-13T01:30:58,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458458) roll requested 2024-11-13T01:30:58,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458465 2024-11-13T01:30:58,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458465 2024-11-13T01:30:58,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458465) roll requested 2024-11-13T01:30:58,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458472 2024-11-13T01:30:58,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458472 2024-11-13T01:30:58,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458472) roll requested 2024-11-13T01:30:58,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458480 2024-11-13T01:30:58,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458480 2024-11-13T01:30:58,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458480) roll requested 2024-11-13T01:30:58,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458488 2024-11-13T01:30:58,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458488 2024-11-13T01:30:58,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458488) roll requested 2024-11-13T01:30:58,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458496 2024-11-13T01:30:58,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458496 2024-11-13T01:30:58,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458496) roll requested 2024-11-13T01:30:58,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458504 2024-11-13T01:30:58,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458504 2024-11-13T01:30:58,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458504) roll requested 2024-11-13T01:30:58,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458513 2024-11-13T01:30:58,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458513 2024-11-13T01:30:58,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458513) roll requested 2024-11-13T01:30:58,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458520 2024-11-13T01:30:58,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458520 2024-11-13T01:30:58,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458520) roll requested 2024-11-13T01:30:58,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458526 2024-11-13T01:30:58,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458526 2024-11-13T01:30:58,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,534 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458526) roll requested 2024-11-13T01:30:58,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458535 2024-11-13T01:30:58,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458535 2024-11-13T01:30:58,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458535) roll requested 2024-11-13T01:30:58,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458543 2024-11-13T01:30:58,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458543 2024-11-13T01:30:58,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458543) roll requested 2024-11-13T01:30:58,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458548 2024-11-13T01:30:58,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458548 2024-11-13T01:30:58,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458548) roll requested 2024-11-13T01:30:58,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458554 2024-11-13T01:30:58,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458554 2024-11-13T01:30:58,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458554) roll requested 2024-11-13T01:30:58,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458562 2024-11-13T01:30:58,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458562 2024-11-13T01:30:58,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458562) roll requested 2024-11-13T01:30:58,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458571 2024-11-13T01:30:58,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458571 2024-11-13T01:30:58,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458571) roll requested 2024-11-13T01:30:58,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458579 2024-11-13T01:30:58,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,584 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458579 2024-11-13T01:30:58,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458579) roll requested 2024-11-13T01:30:58,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458585 2024-11-13T01:30:58,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458585 2024-11-13T01:30:58,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458585) roll requested 2024-11-13T01:30:58,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458593 2024-11-13T01:30:58,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458585 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458593 2024-11-13T01:30:58,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458593) roll requested 2024-11-13T01:30:58,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458602 2024-11-13T01:30:58,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458602 2024-11-13T01:30:58,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458602) roll requested 2024-11-13T01:30:58,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458610 2024-11-13T01:30:58,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458610 2024-11-13T01:30:58,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458610) roll requested 2024-11-13T01:30:58,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458616 2024-11-13T01:30:58,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458616 2024-11-13T01:30:58,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458616) roll requested 2024-11-13T01:30:58,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458623 2024-11-13T01:30:58,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458623 2024-11-13T01:30:58,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458623) roll requested 2024-11-13T01:30:58,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458631 2024-11-13T01:30:58,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458631 2024-11-13T01:30:58,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458631) roll requested 2024-11-13T01:30:58,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458637 2024-11-13T01:30:58,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458637 2024-11-13T01:30:58,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458637) roll requested 2024-11-13T01:30:58,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458644 2024-11-13T01:30:58,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458644 2024-11-13T01:30:58,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458644) roll requested 2024-11-13T01:30:58,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458651 2024-11-13T01:30:58,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458651 2024-11-13T01:30:58,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458651) roll requested 2024-11-13T01:30:58,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458659 2024-11-13T01:30:58,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458659 2024-11-13T01:30:58,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458659) roll requested 2024-11-13T01:30:58,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458668 2024-11-13T01:30:58,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458668 2024-11-13T01:30:58,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458668) roll requested 2024-11-13T01:30:58,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458677 2024-11-13T01:30:58,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458677 2024-11-13T01:30:58,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458677) roll requested 2024-11-13T01:30:58,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458686 2024-11-13T01:30:58,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458686 2024-11-13T01:30:58,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458686) roll requested 2024-11-13T01:30:58,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458719 2024-11-13T01:30:58,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458719 2024-11-13T01:30:58,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458719) roll requested 2024-11-13T01:30:58,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458729 2024-11-13T01:30:58,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458729 2024-11-13T01:30:58,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458729) roll requested 2024-11-13T01:30:58,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458740 2024-11-13T01:30:58,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458729 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458740 2024-11-13T01:30:58,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458740) roll requested 2024-11-13T01:30:58,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458751 2024-11-13T01:30:58,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458751 2024-11-13T01:30:58,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458751) roll requested 2024-11-13T01:30:58,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458756 2024-11-13T01:30:58,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458756 2024-11-13T01:30:58,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458756) roll requested 2024-11-13T01:30:58,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458762 2024-11-13T01:30:58,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458762 2024-11-13T01:30:58,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458762) roll requested 2024-11-13T01:30:58,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458769 2024-11-13T01:30:58,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458769 2024-11-13T01:30:58,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458769) roll requested 2024-11-13T01:30:58,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458776 2024-11-13T01:30:58,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458776 2024-11-13T01:30:58,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458776) roll requested 2024-11-13T01:30:58,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458782 2024-11-13T01:30:58,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458782 2024-11-13T01:30:58,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458782) roll requested 2024-11-13T01:30:58,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458789 2024-11-13T01:30:58,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458789 2024-11-13T01:30:58,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458789) roll requested 2024-11-13T01:30:58,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458797 2024-11-13T01:30:58,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458797 2024-11-13T01:30:58,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458797) roll requested 2024-11-13T01:30:58,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458805 2024-11-13T01:30:58,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458805 2024-11-13T01:30:58,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458805) roll requested 2024-11-13T01:30:58,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458814 2024-11-13T01:30:58,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458814 2024-11-13T01:30:58,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458814) roll requested 2024-11-13T01:30:58,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458822 2024-11-13T01:30:58,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458822 2024-11-13T01:30:58,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458822) roll requested 2024-11-13T01:30:58,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458831 2024-11-13T01:30:58,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458831 2024-11-13T01:30:58,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458831) roll requested 2024-11-13T01:30:58,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458839 2024-11-13T01:30:58,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458839 2024-11-13T01:30:58,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458839) roll requested 2024-11-13T01:30:58,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458847 2024-11-13T01:30:58,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458839 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458847 2024-11-13T01:30:58,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458847) roll requested 2024-11-13T01:30:58,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458855 2024-11-13T01:30:58,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458855 2024-11-13T01:30:58,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458855) roll requested 2024-11-13T01:30:58,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458862 2024-11-13T01:30:58,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458862 2024-11-13T01:30:58,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458862) roll requested 2024-11-13T01:30:58,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458870 2024-11-13T01:30:58,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458870 2024-11-13T01:30:58,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458870) roll requested 2024-11-13T01:30:58,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458878 2024-11-13T01:30:58,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,885 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,885 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458878 2024-11-13T01:30:58,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458878) roll requested 2024-11-13T01:30:58,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458886 2024-11-13T01:30:58,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458886 2024-11-13T01:30:58,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458886) roll requested 2024-11-13T01:30:58,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458891 2024-11-13T01:30:58,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458886 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458891 2024-11-13T01:30:58,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,898 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458891) roll requested 2024-11-13T01:30:58,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458898 2024-11-13T01:30:58,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458898 2024-11-13T01:30:58,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,906 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458898) roll requested 2024-11-13T01:30:58,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458906 2024-11-13T01:30:58,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458906 2024-11-13T01:30:58,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458906) roll requested 2024-11-13T01:30:58,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458915 2024-11-13T01:30:58,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458915 2024-11-13T01:30:58,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458915) roll requested 2024-11-13T01:30:58,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458925 2024-11-13T01:30:58,933 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,933 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458925 2024-11-13T01:30:58,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458925) roll requested 2024-11-13T01:30:58,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458934 2024-11-13T01:30:58,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458934 2024-11-13T01:30:58,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458934) roll requested 2024-11-13T01:30:58,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458944 2024-11-13T01:30:58,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458944 2024-11-13T01:30:58,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458944) roll requested 2024-11-13T01:30:58,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458954 2024-11-13T01:30:58,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458954 2024-11-13T01:30:58,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458954) roll requested 2024-11-13T01:30:58,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458961 2024-11-13T01:30:58,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458961 2024-11-13T01:30:58,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,973 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458961) roll requested 2024-11-13T01:30:58,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458974 2024-11-13T01:30:58,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458974 2024-11-13T01:30:58,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458974) roll requested 2024-11-13T01:30:58,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458981 2024-11-13T01:30:58,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458981 2024-11-13T01:30:58,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458981) roll requested 2024-11-13T01:30:58,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458987 2024-11-13T01:30:58,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:58,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458987 2024-11-13T01:30:58,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:58,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:58,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458987) roll requested 2024-11-13T01:30:58,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461458994 2024-11-13T01:30:59,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458994 2024-11-13T01:30:59,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461458994) roll requested 2024-11-13T01:30:59,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459003 2024-11-13T01:30:59,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461458994 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459003 2024-11-13T01:30:59,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459003) roll requested 2024-11-13T01:30:59,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459009 2024-11-13T01:30:59,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459009 2024-11-13T01:30:59,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459009) roll requested 2024-11-13T01:30:59,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459015 2024-11-13T01:30:59,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459015 2024-11-13T01:30:59,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459015) roll requested 2024-11-13T01:30:59,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459022 2024-11-13T01:30:59,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459022 2024-11-13T01:30:59,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459022) roll requested 2024-11-13T01:30:59,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459029 2024-11-13T01:30:59,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459029 2024-11-13T01:30:59,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459029) roll requested 2024-11-13T01:30:59,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459037 2024-11-13T01:30:59,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459037 2024-11-13T01:30:59,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459037) roll requested 2024-11-13T01:30:59,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459043 2024-11-13T01:30:59,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459043 2024-11-13T01:30:59,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459043) roll requested 2024-11-13T01:30:59,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459051 2024-11-13T01:30:59,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459051 2024-11-13T01:30:59,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459051) roll requested 2024-11-13T01:30:59,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459058 2024-11-13T01:30:59,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459058 2024-11-13T01:30:59,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459058) roll requested 2024-11-13T01:30:59,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459065 2024-11-13T01:30:59,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459065 2024-11-13T01:30:59,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459065) roll requested 2024-11-13T01:30:59,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459073 2024-11-13T01:30:59,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459073 2024-11-13T01:30:59,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459073) roll requested 2024-11-13T01:30:59,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459080 2024-11-13T01:30:59,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459080 2024-11-13T01:30:59,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459080) roll requested 2024-11-13T01:30:59,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459089 2024-11-13T01:30:59,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459089 2024-11-13T01:30:59,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459089) roll requested 2024-11-13T01:30:59,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459097 2024-11-13T01:30:59,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459097 2024-11-13T01:30:59,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,103 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459097) roll requested 2024-11-13T01:30:59,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459103 2024-11-13T01:30:59,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459103 2024-11-13T01:30:59,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459103) roll requested 2024-11-13T01:30:59,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459108 2024-11-13T01:30:59,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459108 2024-11-13T01:30:59,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459108) roll requested 2024-11-13T01:30:59,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459116 2024-11-13T01:30:59,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459116 2024-11-13T01:30:59,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,122 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459116) roll requested 2024-11-13T01:30:59,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459122 2024-11-13T01:30:59,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459122 2024-11-13T01:30:59,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459122) roll requested 2024-11-13T01:30:59,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459130 2024-11-13T01:30:59,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459130 2024-11-13T01:30:59,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459130) roll requested 2024-11-13T01:30:59,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459138 2024-11-13T01:30:59,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459138 2024-11-13T01:30:59,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459138) roll requested 2024-11-13T01:30:59,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459147 2024-11-13T01:30:59,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459147 2024-11-13T01:30:59,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459147) roll requested 2024-11-13T01:30:59,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459155 2024-11-13T01:30:59,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459155 2024-11-13T01:30:59,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459155) roll requested 2024-11-13T01:30:59,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459165 2024-11-13T01:30:59,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459165 2024-11-13T01:30:59,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459165) roll requested 2024-11-13T01:30:59,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459174 2024-11-13T01:30:59,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459174 2024-11-13T01:30:59,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459174) roll requested 2024-11-13T01:30:59,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459183 2024-11-13T01:30:59,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459183 2024-11-13T01:30:59,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459183) roll requested 2024-11-13T01:30:59,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459191 2024-11-13T01:30:59,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459191 2024-11-13T01:30:59,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459191) roll requested 2024-11-13T01:30:59,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459200 2024-11-13T01:30:59,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459200 2024-11-13T01:30:59,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459200) roll requested 2024-11-13T01:30:59,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459210 2024-11-13T01:30:59,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459210 2024-11-13T01:30:59,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459210) roll requested 2024-11-13T01:30:59,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459221 2024-11-13T01:30:59,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459221 2024-11-13T01:30:59,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459221) roll requested 2024-11-13T01:30:59,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459230 2024-11-13T01:30:59,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459230 2024-11-13T01:30:59,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459230) roll requested 2024-11-13T01:30:59,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459241 2024-11-13T01:30:59,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459241 2024-11-13T01:30:59,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459241) roll requested 2024-11-13T01:30:59,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459247 2024-11-13T01:30:59,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459247 2024-11-13T01:30:59,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459247) roll requested 2024-11-13T01:30:59,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459252 2024-11-13T01:30:59,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459252 2024-11-13T01:30:59,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459252) roll requested 2024-11-13T01:30:59,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459259 2024-11-13T01:30:59,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459259 2024-11-13T01:30:59,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459259) roll requested 2024-11-13T01:30:59,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459267 2024-11-13T01:30:59,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459267 2024-11-13T01:30:59,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459267) roll requested 2024-11-13T01:30:59,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459276 2024-11-13T01:30:59,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459276 2024-11-13T01:30:59,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459276) roll requested 2024-11-13T01:30:59,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459287 2024-11-13T01:30:59,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459287 2024-11-13T01:30:59,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459287) roll requested 2024-11-13T01:30:59,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459296 2024-11-13T01:30:59,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459296 2024-11-13T01:30:59,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459296) roll requested 2024-11-13T01:30:59,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459305 2024-11-13T01:30:59,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459296 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459305 2024-11-13T01:30:59,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459305) roll requested 2024-11-13T01:30:59,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459312 2024-11-13T01:30:59,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459312 2024-11-13T01:30:59,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459312) roll requested 2024-11-13T01:30:59,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459321 2024-11-13T01:30:59,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459321 2024-11-13T01:30:59,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459321) roll requested 2024-11-13T01:30:59,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459328 2024-11-13T01:30:59,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459328 2024-11-13T01:30:59,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459328) roll requested 2024-11-13T01:30:59,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459335 2024-11-13T01:30:59,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459335 2024-11-13T01:30:59,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459335) roll requested 2024-11-13T01:30:59,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459341 2024-11-13T01:30:59,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459341 2024-11-13T01:30:59,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459341) roll requested 2024-11-13T01:30:59,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459347 2024-11-13T01:30:59,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459347 2024-11-13T01:30:59,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459347) roll requested 2024-11-13T01:30:59,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459353 2024-11-13T01:30:59,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459353 2024-11-13T01:30:59,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459353) roll requested 2024-11-13T01:30:59,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459358 2024-11-13T01:30:59,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459353 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459358 2024-11-13T01:30:59,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459358) roll requested 2024-11-13T01:30:59,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459363 2024-11-13T01:30:59,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459363 2024-11-13T01:30:59,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459363) roll requested 2024-11-13T01:30:59,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459368 2024-11-13T01:30:59,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459368 2024-11-13T01:30:59,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,374 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459363 is not closed yet, will try archiving it next time 2024-11-13T01:30:59,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459368) roll requested 2024-11-13T01:30:59,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459374 2024-11-13T01:30:59,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459374 2024-11-13T01:30:59,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459374) roll requested 2024-11-13T01:30:59,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459438 2024-11-13T01:30:59,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459374 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459438 2024-11-13T01:30:59,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459438) roll requested 2024-11-13T01:30:59,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459444 2024-11-13T01:30:59,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459444 2024-11-13T01:30:59,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459444) roll requested 2024-11-13T01:30:59,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459451 2024-11-13T01:30:59,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459451 2024-11-13T01:30:59,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459451) roll requested 2024-11-13T01:30:59,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459459 2024-11-13T01:30:59,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459459 2024-11-13T01:30:59,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459459) roll requested 2024-11-13T01:30:59,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459469 2024-11-13T01:30:59,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459469 2024-11-13T01:30:59,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459469) roll requested 2024-11-13T01:30:59,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459479 2024-11-13T01:30:59,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459469 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459479 2024-11-13T01:30:59,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459479) roll requested 2024-11-13T01:30:59,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459484 2024-11-13T01:30:59,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459484 2024-11-13T01:30:59,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459484) roll requested 2024-11-13T01:30:59,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459489 2024-11-13T01:30:59,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459489 2024-11-13T01:30:59,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459489) roll requested 2024-11-13T01:30:59,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459495 2024-11-13T01:30:59,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459495 2024-11-13T01:30:59,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459495) roll requested 2024-11-13T01:30:59,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459501 2024-11-13T01:30:59,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459501 2024-11-13T01:30:59,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459501) roll requested 2024-11-13T01:30:59,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459508 2024-11-13T01:30:59,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459508 2024-11-13T01:30:59,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459508) roll requested 2024-11-13T01:30:59,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459515 2024-11-13T01:30:59,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459515 2024-11-13T01:30:59,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459515) roll requested 2024-11-13T01:30:59,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459520 2024-11-13T01:30:59,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459520 2024-11-13T01:30:59,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,527 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459520) roll requested 2024-11-13T01:30:59,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459527 2024-11-13T01:30:59,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459527 2024-11-13T01:30:59,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459527) roll requested 2024-11-13T01:30:59,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459535 2024-11-13T01:30:59,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459527 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459535 2024-11-13T01:30:59,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459535) roll requested 2024-11-13T01:30:59,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459545 2024-11-13T01:30:59,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459545 2024-11-13T01:30:59,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459545) roll requested 2024-11-13T01:30:59,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459557 2024-11-13T01:30:59,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459557 2024-11-13T01:30:59,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459557) roll requested 2024-11-13T01:30:59,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459563 2024-11-13T01:30:59,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459563 2024-11-13T01:30:59,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459563) roll requested 2024-11-13T01:30:59,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459569 2024-11-13T01:30:59,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459569 2024-11-13T01:30:59,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459569) roll requested 2024-11-13T01:30:59,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459575 2024-11-13T01:30:59,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459575 2024-11-13T01:30:59,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459575) roll requested 2024-11-13T01:30:59,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459581 2024-11-13T01:30:59,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459581 2024-11-13T01:30:59,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459581) roll requested 2024-11-13T01:30:59,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459588 2024-11-13T01:30:59,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459588 2024-11-13T01:30:59,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459588) roll requested 2024-11-13T01:30:59,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459597 2024-11-13T01:30:59,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459597 2024-11-13T01:30:59,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459597) roll requested 2024-11-13T01:30:59,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459605 2024-11-13T01:30:59,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459605 2024-11-13T01:30:59,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459605) roll requested 2024-11-13T01:30:59,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459614 2024-11-13T01:30:59,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459614 2024-11-13T01:30:59,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459614) roll requested 2024-11-13T01:30:59,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459622 2024-11-13T01:30:59,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459622 2024-11-13T01:30:59,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459622) roll requested 2024-11-13T01:30:59,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459628 2024-11-13T01:30:59,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459628 2024-11-13T01:30:59,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459628) roll requested 2024-11-13T01:30:59,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459637 2024-11-13T01:30:59,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459637 2024-11-13T01:30:59,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459637) roll requested 2024-11-13T01:30:59,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459645 2024-11-13T01:30:59,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459645 2024-11-13T01:30:59,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459645) roll requested 2024-11-13T01:30:59,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459654 2024-11-13T01:30:59,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459654 2024-11-13T01:30:59,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459654) roll requested 2024-11-13T01:30:59,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459662 2024-11-13T01:30:59,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459662 2024-11-13T01:30:59,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459662) roll requested 2024-11-13T01:30:59,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459673 2024-11-13T01:30:59,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459673 2024-11-13T01:30:59,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,679 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,679 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459673) roll requested 2024-11-13T01:30:59,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459679 2024-11-13T01:30:59,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459679 2024-11-13T01:30:59,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459679) roll requested 2024-11-13T01:30:59,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459685 2024-11-13T01:30:59,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459679 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459685 2024-11-13T01:30:59,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459685) roll requested 2024-11-13T01:30:59,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459694 2024-11-13T01:30:59,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459694 2024-11-13T01:30:59,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459694) roll requested 2024-11-13T01:30:59,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459699 2024-11-13T01:30:59,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459699 2024-11-13T01:30:59,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459699) roll requested 2024-11-13T01:30:59,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459705 2024-11-13T01:30:59,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459705 2024-11-13T01:30:59,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459705) roll requested 2024-11-13T01:30:59,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459711 2024-11-13T01:30:59,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459711 2024-11-13T01:30:59,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459711) roll requested 2024-11-13T01:30:59,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459719 2024-11-13T01:30:59,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459711 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459719 2024-11-13T01:30:59,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459719) roll requested 2024-11-13T01:30:59,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459725 2024-11-13T01:30:59,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459725 2024-11-13T01:30:59,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459725) roll requested 2024-11-13T01:30:59,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459731 2024-11-13T01:30:59,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459731 2024-11-13T01:30:59,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459731) roll requested 2024-11-13T01:30:59,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459736 2024-11-13T01:30:59,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459736 2024-11-13T01:30:59,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459736) roll requested 2024-11-13T01:30:59,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459741 2024-11-13T01:30:59,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459741 2024-11-13T01:30:59,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459741) roll requested 2024-11-13T01:30:59,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459746 2024-11-13T01:30:59,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459746 2024-11-13T01:30:59,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459746) roll requested 2024-11-13T01:30:59,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459751 2024-11-13T01:30:59,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459751 2024-11-13T01:30:59,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459751) roll requested 2024-11-13T01:30:59,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459757 2024-11-13T01:30:59,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459757 2024-11-13T01:30:59,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459757) roll requested 2024-11-13T01:30:59,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459763 2024-11-13T01:30:59,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459763 2024-11-13T01:30:59,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459763) roll requested 2024-11-13T01:30:59,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459767 2024-11-13T01:30:59,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459767 2024-11-13T01:30:59,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459767) roll requested 2024-11-13T01:30:59,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459773 2024-11-13T01:30:59,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459773 2024-11-13T01:30:59,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459773) roll requested 2024-11-13T01:30:59,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459779 2024-11-13T01:30:59,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459779 2024-11-13T01:30:59,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459779) roll requested 2024-11-13T01:30:59,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459786 2024-11-13T01:30:59,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459786 2024-11-13T01:30:59,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459786) roll requested 2024-11-13T01:30:59,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459796 2024-11-13T01:30:59,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459796 2024-11-13T01:30:59,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459786 is not closed yet, will try archiving it next time 2024-11-13T01:30:59,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459796) roll requested 2024-11-13T01:30:59,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459803 2024-11-13T01:30:59,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459803 2024-11-13T01:30:59,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459803) roll requested 2024-11-13T01:30:59,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459810 2024-11-13T01:30:59,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459810 2024-11-13T01:30:59,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459810) roll requested 2024-11-13T01:30:59,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459819 2024-11-13T01:30:59,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459819 2024-11-13T01:30:59,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459819) roll requested 2024-11-13T01:30:59,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459826 2024-11-13T01:30:59,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459826 2024-11-13T01:30:59,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459826) roll requested 2024-11-13T01:30:59,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459834 2024-11-13T01:30:59,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459834 2024-11-13T01:30:59,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459834) roll requested 2024-11-13T01:30:59,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459843 2024-11-13T01:30:59,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459843 2024-11-13T01:30:59,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459843) roll requested 2024-11-13T01:30:59,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459853 2024-11-13T01:30:59,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459853 2024-11-13T01:30:59,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459853) roll requested 2024-11-13T01:30:59,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459860 2024-11-13T01:30:59,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459860 2024-11-13T01:30:59,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459860) roll requested 2024-11-13T01:30:59,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459867 2024-11-13T01:30:59,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459867 2024-11-13T01:30:59,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459867) roll requested 2024-11-13T01:30:59,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459876 2024-11-13T01:30:59,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459876 2024-11-13T01:30:59,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459876) roll requested 2024-11-13T01:30:59,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459884 2024-11-13T01:30:59,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459884 2024-11-13T01:30:59,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459884) roll requested 2024-11-13T01:30:59,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459890 2024-11-13T01:30:59,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459890 2024-11-13T01:30:59,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459890) roll requested 2024-11-13T01:30:59,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459895 2024-11-13T01:30:59,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459895 2024-11-13T01:30:59,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459895) roll requested 2024-11-13T01:30:59,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459902 2024-11-13T01:30:59,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459902 2024-11-13T01:30:59,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459902) roll requested 2024-11-13T01:30:59,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459910 2024-11-13T01:30:59,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459910 2024-11-13T01:30:59,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459910) roll requested 2024-11-13T01:30:59,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459920 2024-11-13T01:30:59,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459920 2024-11-13T01:30:59,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459920) roll requested 2024-11-13T01:30:59,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459929 2024-11-13T01:30:59,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459920 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459929 2024-11-13T01:30:59,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459929) roll requested 2024-11-13T01:30:59,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459936 2024-11-13T01:30:59,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459936 2024-11-13T01:30:59,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459936) roll requested 2024-11-13T01:30:59,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459944 2024-11-13T01:30:59,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459944 2024-11-13T01:30:59,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459944) roll requested 2024-11-13T01:30:59,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459952 2024-11-13T01:30:59,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459952 2024-11-13T01:30:59,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459952) roll requested 2024-11-13T01:30:59,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459958 2024-11-13T01:30:59,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459958 2024-11-13T01:30:59,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459958) roll requested 2024-11-13T01:30:59,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459964 2024-11-13T01:30:59,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459964 2024-11-13T01:30:59,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459964) roll requested 2024-11-13T01:30:59,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459972 2024-11-13T01:30:59,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459972 2024-11-13T01:30:59,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459972) roll requested 2024-11-13T01:30:59,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459980 2024-11-13T01:30:59,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459980 2024-11-13T01:30:59,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459980) roll requested 2024-11-13T01:30:59,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459988 2024-11-13T01:30:59,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:30:59,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459988 2024-11-13T01:30:59,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:30:59,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:30:59,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459988) roll requested 2024-11-13T01:30:59,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461459997 2024-11-13T01:31:00,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459997 2024-11-13T01:31:00,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461459997) roll requested 2024-11-13T01:31:00,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460002 2024-11-13T01:31:00,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461459997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460002 2024-11-13T01:31:00,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460002) roll requested 2024-11-13T01:31:00,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460008 2024-11-13T01:31:00,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460008 2024-11-13T01:31:00,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460008) roll requested 2024-11-13T01:31:00,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460015 2024-11-13T01:31:00,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,021 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460015 2024-11-13T01:31:00,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460015) roll requested 2024-11-13T01:31:00,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460022 2024-11-13T01:31:00,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460022 2024-11-13T01:31:00,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460022) roll requested 2024-11-13T01:31:00,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460030 2024-11-13T01:31:00,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460030 2024-11-13T01:31:00,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,038 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460030) roll requested 2024-11-13T01:31:00,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460038 2024-11-13T01:31:00,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460038 2024-11-13T01:31:00,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460038) roll requested 2024-11-13T01:31:00,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460048 2024-11-13T01:31:00,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460038 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460048 2024-11-13T01:31:00,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460048) roll requested 2024-11-13T01:31:00,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460056 2024-11-13T01:31:00,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460056 2024-11-13T01:31:00,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460056) roll requested 2024-11-13T01:31:00,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460064 2024-11-13T01:31:00,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460064 2024-11-13T01:31:00,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460064) roll requested 2024-11-13T01:31:00,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460073 2024-11-13T01:31:00,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460073 2024-11-13T01:31:00,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460073) roll requested 2024-11-13T01:31:00,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460082 2024-11-13T01:31:00,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460082 2024-11-13T01:31:00,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460082) roll requested 2024-11-13T01:31:00,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460089 2024-11-13T01:31:00,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460089 2024-11-13T01:31:00,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460089) roll requested 2024-11-13T01:31:00,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460096 2024-11-13T01:31:00,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460096 2024-11-13T01:31:00,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460096) roll requested 2024-11-13T01:31:00,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460103 2024-11-13T01:31:00,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460103 2024-11-13T01:31:00,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,110 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460103) roll requested 2024-11-13T01:31:00,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460111 2024-11-13T01:31:00,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460111 2024-11-13T01:31:00,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460111) roll requested 2024-11-13T01:31:00,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460121 2024-11-13T01:31:00,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460121 2024-11-13T01:31:00,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460121) roll requested 2024-11-13T01:31:00,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460128 2024-11-13T01:31:00,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460128 2024-11-13T01:31:00,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460128) roll requested 2024-11-13T01:31:00,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460136 2024-11-13T01:31:00,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460136 2024-11-13T01:31:00,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460136) roll requested 2024-11-13T01:31:00,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460143 2024-11-13T01:31:00,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460136 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460143 2024-11-13T01:31:00,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460143) roll requested 2024-11-13T01:31:00,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460150 2024-11-13T01:31:00,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460150 2024-11-13T01:31:00,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460150) roll requested 2024-11-13T01:31:00,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460158 2024-11-13T01:31:00,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460158 2024-11-13T01:31:00,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460158) roll requested 2024-11-13T01:31:00,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460167 2024-11-13T01:31:00,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460167 2024-11-13T01:31:00,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460167) roll requested 2024-11-13T01:31:00,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460176 2024-11-13T01:31:00,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460176 2024-11-13T01:31:00,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460176) roll requested 2024-11-13T01:31:00,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460185 2024-11-13T01:31:00,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460185 2024-11-13T01:31:00,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460185) roll requested 2024-11-13T01:31:00,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460194 2024-11-13T01:31:00,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460194 2024-11-13T01:31:00,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460194) roll requested 2024-11-13T01:31:00,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460207 2024-11-13T01:31:00,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460194 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460207 2024-11-13T01:31:00,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460207) roll requested 2024-11-13T01:31:00,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460216 2024-11-13T01:31:00,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460216 2024-11-13T01:31:00,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460216) roll requested 2024-11-13T01:31:00,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460222 2024-11-13T01:31:00,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,227 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460222 2024-11-13T01:31:00,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460222) roll requested 2024-11-13T01:31:00,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460228 2024-11-13T01:31:00,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460228 2024-11-13T01:31:00,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460228) roll requested 2024-11-13T01:31:00,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460233 2024-11-13T01:31:00,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460233 2024-11-13T01:31:00,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460233) roll requested 2024-11-13T01:31:00,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460245 2024-11-13T01:31:00,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460233 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460245 2024-11-13T01:31:00,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460245) roll requested 2024-11-13T01:31:00,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460252 2024-11-13T01:31:00,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460252 2024-11-13T01:31:00,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460252) roll requested 2024-11-13T01:31:00,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460257 2024-11-13T01:31:00,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460257 2024-11-13T01:31:00,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460257) roll requested 2024-11-13T01:31:00,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460262 2024-11-13T01:31:00,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460262 2024-11-13T01:31:00,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460262) roll requested 2024-11-13T01:31:00,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460268 2024-11-13T01:31:00,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460268 2024-11-13T01:31:00,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460268) roll requested 2024-11-13T01:31:00,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460274 2024-11-13T01:31:00,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460274 2024-11-13T01:31:00,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,282 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460274) roll requested 2024-11-13T01:31:00,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460282 2024-11-13T01:31:00,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460282 2024-11-13T01:31:00,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460282) roll requested 2024-11-13T01:31:00,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460290 2024-11-13T01:31:00,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460290 2024-11-13T01:31:00,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460290) roll requested 2024-11-13T01:31:00,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460300 2024-11-13T01:31:00,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460290 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460300 2024-11-13T01:31:00,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460300) roll requested 2024-11-13T01:31:00,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460306 2024-11-13T01:31:00,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460306 2024-11-13T01:31:00,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460306) roll requested 2024-11-13T01:31:00,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460313 2024-11-13T01:31:00,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460313 2024-11-13T01:31:00,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460313) roll requested 2024-11-13T01:31:00,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460322 2024-11-13T01:31:00,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460322 2024-11-13T01:31:00,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460322) roll requested 2024-11-13T01:31:00,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460331 2024-11-13T01:31:00,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460331 2024-11-13T01:31:00,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460331) roll requested 2024-11-13T01:31:00,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460338 2024-11-13T01:31:00,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460338 2024-11-13T01:31:00,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,345 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460338) roll requested 2024-11-13T01:31:00,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460345 2024-11-13T01:31:00,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460345 2024-11-13T01:31:00,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460345) roll requested 2024-11-13T01:31:00,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460356 2024-11-13T01:31:00,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460356 2024-11-13T01:31:00,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,364 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460356) roll requested 2024-11-13T01:31:00,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460364 2024-11-13T01:31:00,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460364 2024-11-13T01:31:00,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460364) roll requested 2024-11-13T01:31:00,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460372 2024-11-13T01:31:00,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460364 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460372 2024-11-13T01:31:00,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460372) roll requested 2024-11-13T01:31:00,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460379 2024-11-13T01:31:00,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460379 2024-11-13T01:31:00,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460379) roll requested 2024-11-13T01:31:00,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460387 2024-11-13T01:31:00,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460387 2024-11-13T01:31:00,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460387) roll requested 2024-11-13T01:31:00,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460400 2024-11-13T01:31:00,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460400 2024-11-13T01:31:00,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460400) roll requested 2024-11-13T01:31:00,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460407 2024-11-13T01:31:00,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460407 2024-11-13T01:31:00,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460407) roll requested 2024-11-13T01:31:00,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460416 2024-11-13T01:31:00,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460416 2024-11-13T01:31:00,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460416) roll requested 2024-11-13T01:31:00,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460422 2024-11-13T01:31:00,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460422 2024-11-13T01:31:00,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,427 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,427 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460422) roll requested 2024-11-13T01:31:00,427 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460427 2024-11-13T01:31:00,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460427 2024-11-13T01:31:00,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460427) roll requested 2024-11-13T01:31:00,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460434 2024-11-13T01:31:00,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,440 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,440 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460427 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460434 2024-11-13T01:31:00,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460434) roll requested 2024-11-13T01:31:00,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460441 2024-11-13T01:31:00,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460441 2024-11-13T01:31:00,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460441) roll requested 2024-11-13T01:31:00,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460449 2024-11-13T01:31:00,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460449 2024-11-13T01:31:00,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460449) roll requested 2024-11-13T01:31:00,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460457 2024-11-13T01:31:00,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460457 2024-11-13T01:31:00,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460457) roll requested 2024-11-13T01:31:00,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460464 2024-11-13T01:31:00,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460464 2024-11-13T01:31:00,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460464) roll requested 2024-11-13T01:31:00,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460473 2024-11-13T01:31:00,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460473 2024-11-13T01:31:00,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460473) roll requested 2024-11-13T01:31:00,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460480 2024-11-13T01:31:00,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460480 2024-11-13T01:31:00,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460480) roll requested 2024-11-13T01:31:00,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460487 2024-11-13T01:31:00,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460487 2024-11-13T01:31:00,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,495 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460487) roll requested 2024-11-13T01:31:00,495 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460495 2024-11-13T01:31:00,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460495 2024-11-13T01:31:00,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460495) roll requested 2024-11-13T01:31:00,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460502 2024-11-13T01:31:00,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460495 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460502 2024-11-13T01:31:00,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460502) roll requested 2024-11-13T01:31:00,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460511 2024-11-13T01:31:00,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460511 2024-11-13T01:31:00,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460511) roll requested 2024-11-13T01:31:00,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460519 2024-11-13T01:31:00,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460519 2024-11-13T01:31:00,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460519) roll requested 2024-11-13T01:31:00,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460529 2024-11-13T01:31:00,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460529 2024-11-13T01:31:00,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,539 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460529) roll requested 2024-11-13T01:31:00,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460539 2024-11-13T01:31:00,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460539 2024-11-13T01:31:00,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460539) roll requested 2024-11-13T01:31:00,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460548 2024-11-13T01:31:00,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460539 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460548 2024-11-13T01:31:00,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460548) roll requested 2024-11-13T01:31:00,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460557 2024-11-13T01:31:00,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460557 2024-11-13T01:31:00,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460557) roll requested 2024-11-13T01:31:00,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460567 2024-11-13T01:31:00,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460567 2024-11-13T01:31:00,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460567) roll requested 2024-11-13T01:31:00,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460575 2024-11-13T01:31:00,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460575 2024-11-13T01:31:00,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460575) roll requested 2024-11-13T01:31:00,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460584 2024-11-13T01:31:00,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460584 2024-11-13T01:31:00,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460584) roll requested 2024-11-13T01:31:00,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460593 2024-11-13T01:31:00,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460593 2024-11-13T01:31:00,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460593) roll requested 2024-11-13T01:31:00,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460599 2024-11-13T01:31:00,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460599 2024-11-13T01:31:00,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460599) roll requested 2024-11-13T01:31:00,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460605 2024-11-13T01:31:00,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460605 2024-11-13T01:31:00,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460605) roll requested 2024-11-13T01:31:00,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460613 2024-11-13T01:31:00,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460613 2024-11-13T01:31:00,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460613) roll requested 2024-11-13T01:31:00,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460621 2024-11-13T01:31:00,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460621 2024-11-13T01:31:00,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460621) roll requested 2024-11-13T01:31:00,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460629 2024-11-13T01:31:00,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460629 2024-11-13T01:31:00,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460629) roll requested 2024-11-13T01:31:00,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460638 2024-11-13T01:31:00,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460638 2024-11-13T01:31:00,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460638) roll requested 2024-11-13T01:31:00,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460648 2024-11-13T01:31:00,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460648 2024-11-13T01:31:00,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,655 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460648) roll requested 2024-11-13T01:31:00,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460655 2024-11-13T01:31:00,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460655 2024-11-13T01:31:00,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460655) roll requested 2024-11-13T01:31:00,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460663 2024-11-13T01:31:00,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460663 2024-11-13T01:31:00,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460663) roll requested 2024-11-13T01:31:00,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460669 2024-11-13T01:31:00,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460669 2024-11-13T01:31:00,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460669) roll requested 2024-11-13T01:31:00,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460675 2024-11-13T01:31:00,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460675 2024-11-13T01:31:00,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460675) roll requested 2024-11-13T01:31:00,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460683 2024-11-13T01:31:00,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460683 2024-11-13T01:31:00,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460683) roll requested 2024-11-13T01:31:00,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460689 2024-11-13T01:31:00,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460689 2024-11-13T01:31:00,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460689) roll requested 2024-11-13T01:31:00,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460696 2024-11-13T01:31:00,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460696 2024-11-13T01:31:00,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460696) roll requested 2024-11-13T01:31:00,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460703 2024-11-13T01:31:00,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460703 2024-11-13T01:31:00,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460703) roll requested 2024-11-13T01:31:00,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460712 2024-11-13T01:31:00,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460712 2024-11-13T01:31:00,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460712) roll requested 2024-11-13T01:31:00,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460720 2024-11-13T01:31:00,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460720 2024-11-13T01:31:00,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460720) roll requested 2024-11-13T01:31:00,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460727 2024-11-13T01:31:00,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460727 2024-11-13T01:31:00,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460727) roll requested 2024-11-13T01:31:00,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460734 2024-11-13T01:31:00,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460734 2024-11-13T01:31:00,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460734) roll requested 2024-11-13T01:31:00,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460742 2024-11-13T01:31:00,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460742 2024-11-13T01:31:00,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460742) roll requested 2024-11-13T01:31:00,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460751 2024-11-13T01:31:00,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460751 2024-11-13T01:31:00,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460751) roll requested 2024-11-13T01:31:00,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460759 2024-11-13T01:31:00,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460759 2024-11-13T01:31:00,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460759) roll requested 2024-11-13T01:31:00,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460768 2024-11-13T01:31:00,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460768 2024-11-13T01:31:00,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460768) roll requested 2024-11-13T01:31:00,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460776 2024-11-13T01:31:00,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460776 2024-11-13T01:31:00,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460776) roll requested 2024-11-13T01:31:00,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460783 2024-11-13T01:31:00,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460783 2024-11-13T01:31:00,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460783) roll requested 2024-11-13T01:31:00,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460793 2024-11-13T01:31:00,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460793 2024-11-13T01:31:00,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460793) roll requested 2024-11-13T01:31:00,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460803 2024-11-13T01:31:00,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460803 2024-11-13T01:31:00,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460803) roll requested 2024-11-13T01:31:00,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460812 2024-11-13T01:31:00,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460812 2024-11-13T01:31:00,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460812) roll requested 2024-11-13T01:31:00,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460822 2024-11-13T01:31:00,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460822 2024-11-13T01:31:00,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460822) roll requested 2024-11-13T01:31:00,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460828 2024-11-13T01:31:00,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460828 2024-11-13T01:31:00,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460828) roll requested 2024-11-13T01:31:00,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460834 2024-11-13T01:31:00,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460834 2024-11-13T01:31:00,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460834) roll requested 2024-11-13T01:31:00,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460841 2024-11-13T01:31:00,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460841 2024-11-13T01:31:00,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460841) roll requested 2024-11-13T01:31:00,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460850 2024-11-13T01:31:00,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460850 2024-11-13T01:31:00,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,858 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460850) roll requested 2024-11-13T01:31:00,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460858 2024-11-13T01:31:00,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460858 2024-11-13T01:31:00,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460858) roll requested 2024-11-13T01:31:00,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460868 2024-11-13T01:31:00,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460868 2024-11-13T01:31:00,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460868) roll requested 2024-11-13T01:31:00,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460877 2024-11-13T01:31:00,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460877 2024-11-13T01:31:00,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460877) roll requested 2024-11-13T01:31:00,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460884 2024-11-13T01:31:00,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460884 2024-11-13T01:31:00,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460884) roll requested 2024-11-13T01:31:00,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460891 2024-11-13T01:31:00,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460891 2024-11-13T01:31:00,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460891) roll requested 2024-11-13T01:31:00,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460899 2024-11-13T01:31:00,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460899 2024-11-13T01:31:00,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460899) roll requested 2024-11-13T01:31:00,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460907 2024-11-13T01:31:00,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460907 2024-11-13T01:31:00,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,915 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460907) roll requested 2024-11-13T01:31:00,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460915 2024-11-13T01:31:00,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460915 2024-11-13T01:31:00,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460915) roll requested 2024-11-13T01:31:00,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460921 2024-11-13T01:31:00,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460915 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460921 2024-11-13T01:31:00,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460921) roll requested 2024-11-13T01:31:00,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460928 2024-11-13T01:31:00,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460928 2024-11-13T01:31:00,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460928) roll requested 2024-11-13T01:31:00,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460935 2024-11-13T01:31:00,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460935 2024-11-13T01:31:00,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460935) roll requested 2024-11-13T01:31:00,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460945 2024-11-13T01:31:00,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460945 2024-11-13T01:31:00,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460945) roll requested 2024-11-13T01:31:00,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460955 2024-11-13T01:31:00,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460955 2024-11-13T01:31:00,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460955) roll requested 2024-11-13T01:31:00,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460965 2024-11-13T01:31:00,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,972 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460965 2024-11-13T01:31:00,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460965) roll requested 2024-11-13T01:31:00,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460974 2024-11-13T01:31:00,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460974 2024-11-13T01:31:00,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460974) roll requested 2024-11-13T01:31:00,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460982 2024-11-13T01:31:00,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460982 2024-11-13T01:31:00,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460982) roll requested 2024-11-13T01:31:00,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460991 2024-11-13T01:31:00,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:00,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460991 2024-11-13T01:31:00,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:00,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:00,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460991) roll requested 2024-11-13T01:31:00,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461460998 2024-11-13T01:31:01,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460998 2024-11-13T01:31:01,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461460998) roll requested 2024-11-13T01:31:01,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461007 2024-11-13T01:31:01,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461460998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461007 2024-11-13T01:31:01,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461007) roll requested 2024-11-13T01:31:01,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461014 2024-11-13T01:31:01,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461014 2024-11-13T01:31:01,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461014) roll requested 2024-11-13T01:31:01,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461021 2024-11-13T01:31:01,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461021 2024-11-13T01:31:01,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461021) roll requested 2024-11-13T01:31:01,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461031 2024-11-13T01:31:01,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461031 2024-11-13T01:31:01,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461031) roll requested 2024-11-13T01:31:01,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461040 2024-11-13T01:31:01,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461040 2024-11-13T01:31:01,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461040) roll requested 2024-11-13T01:31:01,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461047 2024-11-13T01:31:01,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461047 2024-11-13T01:31:01,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461047) roll requested 2024-11-13T01:31:01,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461055 2024-11-13T01:31:01,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461055 2024-11-13T01:31:01,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461055) roll requested 2024-11-13T01:31:01,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461064 2024-11-13T01:31:01,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461064 2024-11-13T01:31:01,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461064) roll requested 2024-11-13T01:31:01,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461073 2024-11-13T01:31:01,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461073 2024-11-13T01:31:01,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,081 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461073) roll requested 2024-11-13T01:31:01,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461081 2024-11-13T01:31:01,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461081 2024-11-13T01:31:01,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461081) roll requested 2024-11-13T01:31:01,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461086 2024-11-13T01:31:01,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461081 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461086 2024-11-13T01:31:01,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461086) roll requested 2024-11-13T01:31:01,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461092 2024-11-13T01:31:01,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461092 2024-11-13T01:31:01,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461092) roll requested 2024-11-13T01:31:01,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461098 2024-11-13T01:31:01,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461098 2024-11-13T01:31:01,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461098) roll requested 2024-11-13T01:31:01,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461105 2024-11-13T01:31:01,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461105 2024-11-13T01:31:01,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461105) roll requested 2024-11-13T01:31:01,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461112 2024-11-13T01:31:01,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461112 2024-11-13T01:31:01,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461112) roll requested 2024-11-13T01:31:01,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461120 2024-11-13T01:31:01,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461120 2024-11-13T01:31:01,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461120) roll requested 2024-11-13T01:31:01,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461128 2024-11-13T01:31:01,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461128 2024-11-13T01:31:01,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461128) roll requested 2024-11-13T01:31:01,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461135 2024-11-13T01:31:01,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461135 2024-11-13T01:31:01,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461135) roll requested 2024-11-13T01:31:01,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461143 2024-11-13T01:31:01,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461143 2024-11-13T01:31:01,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461143) roll requested 2024-11-13T01:31:01,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461153 2024-11-13T01:31:01,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461153 2024-11-13T01:31:01,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461153) roll requested 2024-11-13T01:31:01,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461162 2024-11-13T01:31:01,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461162 2024-11-13T01:31:01,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461162) roll requested 2024-11-13T01:31:01,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461171 2024-11-13T01:31:01,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461171 2024-11-13T01:31:01,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461171) roll requested 2024-11-13T01:31:01,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461181 2024-11-13T01:31:01,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461181 2024-11-13T01:31:01,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461181) roll requested 2024-11-13T01:31:01,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461186 2024-11-13T01:31:01,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461186 2024-11-13T01:31:01,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461186) roll requested 2024-11-13T01:31:01,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461192 2024-11-13T01:31:01,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461192 2024-11-13T01:31:01,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461192) roll requested 2024-11-13T01:31:01,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461200 2024-11-13T01:31:01,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461200 2024-11-13T01:31:01,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461200) roll requested 2024-11-13T01:31:01,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461206 2024-11-13T01:31:01,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461206 2024-11-13T01:31:01,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461206) roll requested 2024-11-13T01:31:01,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461212 2024-11-13T01:31:01,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461212 2024-11-13T01:31:01,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461212) roll requested 2024-11-13T01:31:01,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461219 2024-11-13T01:31:01,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461219 2024-11-13T01:31:01,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461219) roll requested 2024-11-13T01:31:01,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461227 2024-11-13T01:31:01,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461227 2024-11-13T01:31:01,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461227) roll requested 2024-11-13T01:31:01,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461236 2024-11-13T01:31:01,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461236 2024-11-13T01:31:01,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461236) roll requested 2024-11-13T01:31:01,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461242 2024-11-13T01:31:01,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461242 2024-11-13T01:31:01,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461242) roll requested 2024-11-13T01:31:01,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461248 2024-11-13T01:31:01,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461248 2024-11-13T01:31:01,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461248) roll requested 2024-11-13T01:31:01,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461258 2024-11-13T01:31:01,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461258 2024-11-13T01:31:01,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461258) roll requested 2024-11-13T01:31:01,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461268 2024-11-13T01:31:01,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461268 2024-11-13T01:31:01,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461268) roll requested 2024-11-13T01:31:01,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461274 2024-11-13T01:31:01,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461274 2024-11-13T01:31:01,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461274) roll requested 2024-11-13T01:31:01,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461279 2024-11-13T01:31:01,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461279 2024-11-13T01:31:01,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461279) roll requested 2024-11-13T01:31:01,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461283 2024-11-13T01:31:01,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461283 2024-11-13T01:31:01,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461283) roll requested 2024-11-13T01:31:01,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461288 2024-11-13T01:31:01,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461288 2024-11-13T01:31:01,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,295 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461288) roll requested 2024-11-13T01:31:01,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461295 2024-11-13T01:31:01,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461295 2024-11-13T01:31:01,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461295) roll requested 2024-11-13T01:31:01,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461302 2024-11-13T01:31:01,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461302 2024-11-13T01:31:01,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461302) roll requested 2024-11-13T01:31:01,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461311 2024-11-13T01:31:01,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461311 2024-11-13T01:31:01,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461311) roll requested 2024-11-13T01:31:01,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461319 2024-11-13T01:31:01,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461319 2024-11-13T01:31:01,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461319) roll requested 2024-11-13T01:31:01,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461328 2024-11-13T01:31:01,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461328 2024-11-13T01:31:01,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461328) roll requested 2024-11-13T01:31:01,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461337 2024-11-13T01:31:01,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461337 2024-11-13T01:31:01,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461337) roll requested 2024-11-13T01:31:01,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461343 2024-11-13T01:31:01,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461343 2024-11-13T01:31:01,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461343) roll requested 2024-11-13T01:31:01,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461350 2024-11-13T01:31:01,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461350 2024-11-13T01:31:01,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461350) roll requested 2024-11-13T01:31:01,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461359 2024-11-13T01:31:01,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461359 2024-11-13T01:31:01,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461359) roll requested 2024-11-13T01:31:01,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461368 2024-11-13T01:31:01,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461368 2024-11-13T01:31:01,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461368) roll requested 2024-11-13T01:31:01,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461376 2024-11-13T01:31:01,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461376 2024-11-13T01:31:01,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461376) roll requested 2024-11-13T01:31:01,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461383 2024-11-13T01:31:01,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,391 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,391 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461383 2024-11-13T01:31:01,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461383) roll requested 2024-11-13T01:31:01,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461393 2024-11-13T01:31:01,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461393 2024-11-13T01:31:01,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461393) roll requested 2024-11-13T01:31:01,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461401 2024-11-13T01:31:01,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461401 2024-11-13T01:31:01,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461401) roll requested 2024-11-13T01:31:01,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461409 2024-11-13T01:31:01,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461409 2024-11-13T01:31:01,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461409) roll requested 2024-11-13T01:31:01,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461417 2024-11-13T01:31:01,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461417 2024-11-13T01:31:01,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461417) roll requested 2024-11-13T01:31:01,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461426 2024-11-13T01:31:01,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461426 2024-11-13T01:31:01,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461426) roll requested 2024-11-13T01:31:01,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461436 2024-11-13T01:31:01,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461436 2024-11-13T01:31:01,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461436) roll requested 2024-11-13T01:31:01,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461444 2024-11-13T01:31:01,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461444 2024-11-13T01:31:01,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461444) roll requested 2024-11-13T01:31:01,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461451 2024-11-13T01:31:01,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461444 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461451 2024-11-13T01:31:01,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461451) roll requested 2024-11-13T01:31:01,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461459 2024-11-13T01:31:01,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461459 2024-11-13T01:31:01,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461459) roll requested 2024-11-13T01:31:01,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461467 2024-11-13T01:31:01,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461467 2024-11-13T01:31:01,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461467) roll requested 2024-11-13T01:31:01,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461476 2024-11-13T01:31:01,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461476 2024-11-13T01:31:01,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461476) roll requested 2024-11-13T01:31:01,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461486 2024-11-13T01:31:01,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461486 2024-11-13T01:31:01,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461486) roll requested 2024-11-13T01:31:01,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461492 2024-11-13T01:31:01,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461492 2024-11-13T01:31:01,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461492) roll requested 2024-11-13T01:31:01,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461502 2024-11-13T01:31:01,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461502 2024-11-13T01:31:01,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461502) roll requested 2024-11-13T01:31:01,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461509 2024-11-13T01:31:01,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461509 2024-11-13T01:31:01,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461509) roll requested 2024-11-13T01:31:01,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461515 2024-11-13T01:31:01,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461515 2024-11-13T01:31:01,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461515) roll requested 2024-11-13T01:31:01,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461521 2024-11-13T01:31:01,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461521 2024-11-13T01:31:01,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461521) roll requested 2024-11-13T01:31:01,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461526 2024-11-13T01:31:01,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461526 2024-11-13T01:31:01,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461526) roll requested 2024-11-13T01:31:01,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461532 2024-11-13T01:31:01,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461532 2024-11-13T01:31:01,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461532) roll requested 2024-11-13T01:31:01,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461537 2024-11-13T01:31:01,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461537 2024-11-13T01:31:01,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461537) roll requested 2024-11-13T01:31:01,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461544 2024-11-13T01:31:01,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461544 2024-11-13T01:31:01,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461544) roll requested 2024-11-13T01:31:01,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461551 2024-11-13T01:31:01,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461551 2024-11-13T01:31:01,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461551) roll requested 2024-11-13T01:31:01,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461559 2024-11-13T01:31:01,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461559 2024-11-13T01:31:01,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461559) roll requested 2024-11-13T01:31:01,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461569 2024-11-13T01:31:01,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461569 2024-11-13T01:31:01,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461569) roll requested 2024-11-13T01:31:01,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461579 2024-11-13T01:31:01,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461579 2024-11-13T01:31:01,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461579) roll requested 2024-11-13T01:31:01,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461586 2024-11-13T01:31:01,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461586 2024-11-13T01:31:01,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461586) roll requested 2024-11-13T01:31:01,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461596 2024-11-13T01:31:01,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461596 2024-11-13T01:31:01,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461596) roll requested 2024-11-13T01:31:01,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461605 2024-11-13T01:31:01,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461605 2024-11-13T01:31:01,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461605) roll requested 2024-11-13T01:31:01,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461610 2024-11-13T01:31:01,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461610 2024-11-13T01:31:01,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461610) roll requested 2024-11-13T01:31:01,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461615 2024-11-13T01:31:01,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461615 2024-11-13T01:31:01,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461615) roll requested 2024-11-13T01:31:01,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461620 2024-11-13T01:31:01,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461620 2024-11-13T01:31:01,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461620) roll requested 2024-11-13T01:31:01,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461627 2024-11-13T01:31:01,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461627 2024-11-13T01:31:01,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461627) roll requested 2024-11-13T01:31:01,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461634 2024-11-13T01:31:01,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461634 2024-11-13T01:31:01,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,640 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461634) roll requested 2024-11-13T01:31:01,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461641 2024-11-13T01:31:01,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461634 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461641 2024-11-13T01:31:01,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461641) roll requested 2024-11-13T01:31:01,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461648 2024-11-13T01:31:01,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461648 2024-11-13T01:31:01,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461648) roll requested 2024-11-13T01:31:01,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461654 2024-11-13T01:31:01,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461654 2024-11-13T01:31:01,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461654) roll requested 2024-11-13T01:31:01,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461659 2024-11-13T01:31:01,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461659 2024-11-13T01:31:01,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461659) roll requested 2024-11-13T01:31:01,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461667 2024-11-13T01:31:01,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461667 2024-11-13T01:31:01,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461667) roll requested 2024-11-13T01:31:01,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461676 2024-11-13T01:31:01,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461676 2024-11-13T01:31:01,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461676) roll requested 2024-11-13T01:31:01,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461682 2024-11-13T01:31:01,687 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461682 2024-11-13T01:31:01,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461682) roll requested 2024-11-13T01:31:01,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461690 2024-11-13T01:31:01,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461690 2024-11-13T01:31:01,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461690) roll requested 2024-11-13T01:31:01,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461698 2024-11-13T01:31:01,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461698 2024-11-13T01:31:01,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461698) roll requested 2024-11-13T01:31:01,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461707 2024-11-13T01:31:01,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461707 2024-11-13T01:31:01,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461707) roll requested 2024-11-13T01:31:01,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461713 2024-11-13T01:31:01,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461713 2024-11-13T01:31:01,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461713) roll requested 2024-11-13T01:31:01,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461719 2024-11-13T01:31:01,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461719 2024-11-13T01:31:01,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461719) roll requested 2024-11-13T01:31:01,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461726 2024-11-13T01:31:01,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461726 2024-11-13T01:31:01,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461726) roll requested 2024-11-13T01:31:01,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461734 2024-11-13T01:31:01,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461734 2024-11-13T01:31:01,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461734) roll requested 2024-11-13T01:31:01,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461740 2024-11-13T01:31:01,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461740 2024-11-13T01:31:01,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461740) roll requested 2024-11-13T01:31:01,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461746 2024-11-13T01:31:01,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461746 2024-11-13T01:31:01,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461746) roll requested 2024-11-13T01:31:01,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461753 2024-11-13T01:31:01,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461753 2024-11-13T01:31:01,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461753) roll requested 2024-11-13T01:31:01,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461761 2024-11-13T01:31:01,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461761 2024-11-13T01:31:01,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461761) roll requested 2024-11-13T01:31:01,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461768 2024-11-13T01:31:01,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461768 2024-11-13T01:31:01,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461768) roll requested 2024-11-13T01:31:01,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461775 2024-11-13T01:31:01,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461775 2024-11-13T01:31:01,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,782 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461775) roll requested 2024-11-13T01:31:01,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461783 2024-11-13T01:31:01,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461783 2024-11-13T01:31:01,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,794 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461783) roll requested 2024-11-13T01:31:01,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461795 2024-11-13T01:31:01,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461795 2024-11-13T01:31:01,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461795) roll requested 2024-11-13T01:31:01,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461800 2024-11-13T01:31:01,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461800 2024-11-13T01:31:01,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461800) roll requested 2024-11-13T01:31:01,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461805 2024-11-13T01:31:01,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461805 2024-11-13T01:31:01,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461805) roll requested 2024-11-13T01:31:01,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461810 2024-11-13T01:31:01,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461810 2024-11-13T01:31:01,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461810) roll requested 2024-11-13T01:31:01,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461817 2024-11-13T01:31:01,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461817 2024-11-13T01:31:01,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461817) roll requested 2024-11-13T01:31:01,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461826 2024-11-13T01:31:01,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461826 2024-11-13T01:31:01,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461826) roll requested 2024-11-13T01:31:01,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461834 2024-11-13T01:31:01,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461834 2024-11-13T01:31:01,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461834) roll requested 2024-11-13T01:31:01,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461840 2024-11-13T01:31:01,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461840 2024-11-13T01:31:01,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461840) roll requested 2024-11-13T01:31:01,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461847 2024-11-13T01:31:01,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461847 2024-11-13T01:31:01,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461847) roll requested 2024-11-13T01:31:01,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461856 2024-11-13T01:31:01,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461856 2024-11-13T01:31:01,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461856) roll requested 2024-11-13T01:31:01,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461864 2024-11-13T01:31:01,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461864 2024-11-13T01:31:01,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461864) roll requested 2024-11-13T01:31:01,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461873 2024-11-13T01:31:01,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461873 2024-11-13T01:31:01,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461873) roll requested 2024-11-13T01:31:01,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461882 2024-11-13T01:31:01,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461882 2024-11-13T01:31:01,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461882) roll requested 2024-11-13T01:31:01,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461893 2024-11-13T01:31:01,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461893 2024-11-13T01:31:01,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,900 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461893) roll requested 2024-11-13T01:31:01,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461900 2024-11-13T01:31:01,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461900 2024-11-13T01:31:01,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461900) roll requested 2024-11-13T01:31:01,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461907 2024-11-13T01:31:01,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461907 2024-11-13T01:31:01,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461907) roll requested 2024-11-13T01:31:01,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461916 2024-11-13T01:31:01,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461916 2024-11-13T01:31:01,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461916) roll requested 2024-11-13T01:31:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461925 2024-11-13T01:31:01,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461925 2024-11-13T01:31:01,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461925) roll requested 2024-11-13T01:31:01,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461933 2024-11-13T01:31:01,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461933 2024-11-13T01:31:01,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461933) roll requested 2024-11-13T01:31:01,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461940 2024-11-13T01:31:01,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461940 2024-11-13T01:31:01,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461940) roll requested 2024-11-13T01:31:01,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461947 2024-11-13T01:31:01,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461947 2024-11-13T01:31:01,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461947) roll requested 2024-11-13T01:31:01,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461955 2024-11-13T01:31:01,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461955 2024-11-13T01:31:01,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,963 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461955) roll requested 2024-11-13T01:31:01,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461964 2024-11-13T01:31:01,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461964 2024-11-13T01:31:01,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461964) roll requested 2024-11-13T01:31:01,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461972 2024-11-13T01:31:01,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461964 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461972 2024-11-13T01:31:01,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461972) roll requested 2024-11-13T01:31:01,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461979 2024-11-13T01:31:01,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461979 2024-11-13T01:31:01,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461979) roll requested 2024-11-13T01:31:01,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461988 2024-11-13T01:31:01,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:01,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461988 2024-11-13T01:31:01,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:01,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:01,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461988) roll requested 2024-11-13T01:31:01,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461461997 2024-11-13T01:31:02,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461997 2024-11-13T01:31:02,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461461997) roll requested 2024-11-13T01:31:02,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462006 2024-11-13T01:31:02,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461461997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462006 2024-11-13T01:31:02,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462006) roll requested 2024-11-13T01:31:02,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462015 2024-11-13T01:31:02,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462015 2024-11-13T01:31:02,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462015) roll requested 2024-11-13T01:31:02,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462025 2024-11-13T01:31:02,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462015 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462025 2024-11-13T01:31:02,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462025) roll requested 2024-11-13T01:31:02,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462034 2024-11-13T01:31:02,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462034 2024-11-13T01:31:02,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462034) roll requested 2024-11-13T01:31:02,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462042 2024-11-13T01:31:02,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462042 2024-11-13T01:31:02,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462042) roll requested 2024-11-13T01:31:02,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462050 2024-11-13T01:31:02,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462050 2024-11-13T01:31:02,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462050) roll requested 2024-11-13T01:31:02,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462060 2024-11-13T01:31:02,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462060 2024-11-13T01:31:02,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462060) roll requested 2024-11-13T01:31:02,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462068 2024-11-13T01:31:02,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462068 2024-11-13T01:31:02,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462068) roll requested 2024-11-13T01:31:02,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462077 2024-11-13T01:31:02,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462077 2024-11-13T01:31:02,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462077) roll requested 2024-11-13T01:31:02,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462083 2024-11-13T01:31:02,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462083 2024-11-13T01:31:02,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462083) roll requested 2024-11-13T01:31:02,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462087 2024-11-13T01:31:02,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462087 2024-11-13T01:31:02,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462087) roll requested 2024-11-13T01:31:02,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462093 2024-11-13T01:31:02,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462093 2024-11-13T01:31:02,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462093) roll requested 2024-11-13T01:31:02,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462099 2024-11-13T01:31:02,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462099 2024-11-13T01:31:02,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462099) roll requested 2024-11-13T01:31:02,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462106 2024-11-13T01:31:02,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462106 2024-11-13T01:31:02,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462106) roll requested 2024-11-13T01:31:02,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462113 2024-11-13T01:31:02,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462113 2024-11-13T01:31:02,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462113) roll requested 2024-11-13T01:31:02,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462121 2024-11-13T01:31:02,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462121 2024-11-13T01:31:02,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462121) roll requested 2024-11-13T01:31:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462129 2024-11-13T01:31:02,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462129 2024-11-13T01:31:02,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462129) roll requested 2024-11-13T01:31:02,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462138 2024-11-13T01:31:02,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462138 2024-11-13T01:31:02,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462138) roll requested 2024-11-13T01:31:02,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462146 2024-11-13T01:31:02,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462146 2024-11-13T01:31:02,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462146) roll requested 2024-11-13T01:31:02,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462152 2024-11-13T01:31:02,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462152 2024-11-13T01:31:02,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462152) roll requested 2024-11-13T01:31:02,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462159 2024-11-13T01:31:02,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462159 2024-11-13T01:31:02,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462159) roll requested 2024-11-13T01:31:02,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462167 2024-11-13T01:31:02,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462167 2024-11-13T01:31:02,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462167) roll requested 2024-11-13T01:31:02,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462176 2024-11-13T01:31:02,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462176 2024-11-13T01:31:02,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462176) roll requested 2024-11-13T01:31:02,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462185 2024-11-13T01:31:02,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462185 2024-11-13T01:31:02,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462185) roll requested 2024-11-13T01:31:02,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462191 2024-11-13T01:31:02,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462191 2024-11-13T01:31:02,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462191) roll requested 2024-11-13T01:31:02,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462198 2024-11-13T01:31:02,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462198 2024-11-13T01:31:02,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462198) roll requested 2024-11-13T01:31:02,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462207 2024-11-13T01:31:02,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462207 2024-11-13T01:31:02,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462207) roll requested 2024-11-13T01:31:02,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462216 2024-11-13T01:31:02,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462216 2024-11-13T01:31:02,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462216) roll requested 2024-11-13T01:31:02,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462226 2024-11-13T01:31:02,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462226 2024-11-13T01:31:02,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462226) roll requested 2024-11-13T01:31:02,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462236 2024-11-13T01:31:02,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462236 2024-11-13T01:31:02,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462236) roll requested 2024-11-13T01:31:02,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462244 2024-11-13T01:31:02,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462244 2024-11-13T01:31:02,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462244) roll requested 2024-11-13T01:31:02,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462252 2024-11-13T01:31:02,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462252 2024-11-13T01:31:02,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462252) roll requested 2024-11-13T01:31:02,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462259 2024-11-13T01:31:02,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462259 2024-11-13T01:31:02,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,266 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462259) roll requested 2024-11-13T01:31:02,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462267 2024-11-13T01:31:02,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462267 2024-11-13T01:31:02,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462267) roll requested 2024-11-13T01:31:02,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462275 2024-11-13T01:31:02,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462275 2024-11-13T01:31:02,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462275) roll requested 2024-11-13T01:31:02,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462283 2024-11-13T01:31:02,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462283 2024-11-13T01:31:02,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462283) roll requested 2024-11-13T01:31:02,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462292 2024-11-13T01:31:02,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462292 2024-11-13T01:31:02,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462292) roll requested 2024-11-13T01:31:02,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462300 2024-11-13T01:31:02,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462300 2024-11-13T01:31:02,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462300) roll requested 2024-11-13T01:31:02,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462310 2024-11-13T01:31:02,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462310 2024-11-13T01:31:02,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462310) roll requested 2024-11-13T01:31:02,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462320 2024-11-13T01:31:02,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462320 2024-11-13T01:31:02,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462320) roll requested 2024-11-13T01:31:02,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462326 2024-11-13T01:31:02,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462326 2024-11-13T01:31:02,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,333 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462326) roll requested 2024-11-13T01:31:02,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462333 2024-11-13T01:31:02,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462333 2024-11-13T01:31:02,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462333) roll requested 2024-11-13T01:31:02,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462341 2024-11-13T01:31:02,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462333 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462341 2024-11-13T01:31:02,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462341) roll requested 2024-11-13T01:31:02,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462350 2024-11-13T01:31:02,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462341 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462350 2024-11-13T01:31:02,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462350) roll requested 2024-11-13T01:31:02,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462359 2024-11-13T01:31:02,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462359 2024-11-13T01:31:02,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462359) roll requested 2024-11-13T01:31:02,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462367 2024-11-13T01:31:02,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462367 2024-11-13T01:31:02,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462367) roll requested 2024-11-13T01:31:02,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462375 2024-11-13T01:31:02,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462375 2024-11-13T01:31:02,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462375) roll requested 2024-11-13T01:31:02,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462385 2024-11-13T01:31:02,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462385 2024-11-13T01:31:02,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462385) roll requested 2024-11-13T01:31:02,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462396 2024-11-13T01:31:02,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462396 2024-11-13T01:31:02,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462396) roll requested 2024-11-13T01:31:02,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462403 2024-11-13T01:31:02,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462403 2024-11-13T01:31:02,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462403) roll requested 2024-11-13T01:31:02,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462410 2024-11-13T01:31:02,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462410 2024-11-13T01:31:02,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462410) roll requested 2024-11-13T01:31:02,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462422 2024-11-13T01:31:02,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462422 2024-11-13T01:31:02,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462422) roll requested 2024-11-13T01:31:02,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462431 2024-11-13T01:31:02,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462431 2024-11-13T01:31:02,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462431) roll requested 2024-11-13T01:31:02,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462439 2024-11-13T01:31:02,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462439 2024-11-13T01:31:02,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462439) roll requested 2024-11-13T01:31:02,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462447 2024-11-13T01:31:02,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462447 2024-11-13T01:31:02,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,454 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462447) roll requested 2024-11-13T01:31:02,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462454 2024-11-13T01:31:02,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462454 2024-11-13T01:31:02,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462454) roll requested 2024-11-13T01:31:02,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462463 2024-11-13T01:31:02,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462463 2024-11-13T01:31:02,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462463) roll requested 2024-11-13T01:31:02,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462473 2024-11-13T01:31:02,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462473 2024-11-13T01:31:02,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462473) roll requested 2024-11-13T01:31:02,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462483 2024-11-13T01:31:02,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462483 2024-11-13T01:31:02,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462483) roll requested 2024-11-13T01:31:02,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462490 2024-11-13T01:31:02,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462483 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462490 2024-11-13T01:31:02,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462490) roll requested 2024-11-13T01:31:02,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462496 2024-11-13T01:31:02,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462496 2024-11-13T01:31:02,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462496) roll requested 2024-11-13T01:31:02,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462502 2024-11-13T01:31:02,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462502 2024-11-13T01:31:02,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462502) roll requested 2024-11-13T01:31:02,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462509 2024-11-13T01:31:02,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462509 2024-11-13T01:31:02,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462509) roll requested 2024-11-13T01:31:02,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462516 2024-11-13T01:31:02,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462516 2024-11-13T01:31:02,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462516) roll requested 2024-11-13T01:31:02,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462523 2024-11-13T01:31:02,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462523 2024-11-13T01:31:02,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462523) roll requested 2024-11-13T01:31:02,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462532 2024-11-13T01:31:02,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462532 2024-11-13T01:31:02,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462532) roll requested 2024-11-13T01:31:02,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462542 2024-11-13T01:31:02,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462542 2024-11-13T01:31:02,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462542) roll requested 2024-11-13T01:31:02,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462553 2024-11-13T01:31:02,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462553 2024-11-13T01:31:02,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462553) roll requested 2024-11-13T01:31:02,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462561 2024-11-13T01:31:02,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462561 2024-11-13T01:31:02,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462561) roll requested 2024-11-13T01:31:02,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462569 2024-11-13T01:31:02,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462569 2024-11-13T01:31:02,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462569) roll requested 2024-11-13T01:31:02,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462580 2024-11-13T01:31:02,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462580 2024-11-13T01:31:02,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462580) roll requested 2024-11-13T01:31:02,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462589 2024-11-13T01:31:02,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462589 2024-11-13T01:31:02,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462589) roll requested 2024-11-13T01:31:02,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462595 2024-11-13T01:31:02,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462595 2024-11-13T01:31:02,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462595) roll requested 2024-11-13T01:31:02,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462602 2024-11-13T01:31:02,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462602 2024-11-13T01:31:02,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462602) roll requested 2024-11-13T01:31:02,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462609 2024-11-13T01:31:02,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462609 2024-11-13T01:31:02,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,614 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462609) roll requested 2024-11-13T01:31:02,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462614 2024-11-13T01:31:02,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462614 2024-11-13T01:31:02,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462614) roll requested 2024-11-13T01:31:02,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462620 2024-11-13T01:31:02,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462620 2024-11-13T01:31:02,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462620) roll requested 2024-11-13T01:31:02,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462625 2024-11-13T01:31:02,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462625 2024-11-13T01:31:02,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462625) roll requested 2024-11-13T01:31:02,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462630 2024-11-13T01:31:02,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462630 2024-11-13T01:31:02,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462630) roll requested 2024-11-13T01:31:02,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462636 2024-11-13T01:31:02,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462636 2024-11-13T01:31:02,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462636) roll requested 2024-11-13T01:31:02,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462644 2024-11-13T01:31:02,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462644 2024-11-13T01:31:02,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462644) roll requested 2024-11-13T01:31:02,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462650 2024-11-13T01:31:02,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462650 2024-11-13T01:31:02,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462650) roll requested 2024-11-13T01:31:02,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462655 2024-11-13T01:31:02,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462655 2024-11-13T01:31:02,660 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,660 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462655) roll requested 2024-11-13T01:31:02,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462660 2024-11-13T01:31:02,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462660 2024-11-13T01:31:02,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462660) roll requested 2024-11-13T01:31:02,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462671 2024-11-13T01:31:02,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462671 2024-11-13T01:31:02,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462671) roll requested 2024-11-13T01:31:02,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462676 2024-11-13T01:31:02,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462676 2024-11-13T01:31:02,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462676) roll requested 2024-11-13T01:31:02,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462682 2024-11-13T01:31:02,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462682 2024-11-13T01:31:02,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462682) roll requested 2024-11-13T01:31:02,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462689 2024-11-13T01:31:02,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462689 2024-11-13T01:31:02,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462689) roll requested 2024-11-13T01:31:02,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462695 2024-11-13T01:31:02,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462695 2024-11-13T01:31:02,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462695) roll requested 2024-11-13T01:31:02,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462700 2024-11-13T01:31:02,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462700 2024-11-13T01:31:02,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462700) roll requested 2024-11-13T01:31:02,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462706 2024-11-13T01:31:02,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462706 2024-11-13T01:31:02,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462706) roll requested 2024-11-13T01:31:02,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462712 2024-11-13T01:31:02,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462712 2024-11-13T01:31:02,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462712) roll requested 2024-11-13T01:31:02,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462720 2024-11-13T01:31:02,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462720 2024-11-13T01:31:02,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462720) roll requested 2024-11-13T01:31:02,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462728 2024-11-13T01:31:02,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462728 2024-11-13T01:31:02,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462728) roll requested 2024-11-13T01:31:02,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462736 2024-11-13T01:31:02,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462736 2024-11-13T01:31:02,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462736) roll requested 2024-11-13T01:31:02,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462742 2024-11-13T01:31:02,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462742 2024-11-13T01:31:02,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462742) roll requested 2024-11-13T01:31:02,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462748 2024-11-13T01:31:02,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462748 2024-11-13T01:31:02,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462748) roll requested 2024-11-13T01:31:02,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462753 2024-11-13T01:31:02,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462753 2024-11-13T01:31:02,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462753) roll requested 2024-11-13T01:31:02,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462759 2024-11-13T01:31:02,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462759 2024-11-13T01:31:02,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462759) roll requested 2024-11-13T01:31:02,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462765 2024-11-13T01:31:02,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462765 2024-11-13T01:31:02,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462765) roll requested 2024-11-13T01:31:02,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462770 2024-11-13T01:31:02,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462770 2024-11-13T01:31:02,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462770) roll requested 2024-11-13T01:31:02,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462775 2024-11-13T01:31:02,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462775 2024-11-13T01:31:02,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462775) roll requested 2024-11-13T01:31:02,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462780 2024-11-13T01:31:02,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462775 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462780 2024-11-13T01:31:02,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462780) roll requested 2024-11-13T01:31:02,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462786 2024-11-13T01:31:02,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462786 2024-11-13T01:31:02,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462786) roll requested 2024-11-13T01:31:02,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462793 2024-11-13T01:31:02,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462793 2024-11-13T01:31:02,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462793) roll requested 2024-11-13T01:31:02,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462800 2024-11-13T01:31:02,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462800 2024-11-13T01:31:02,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462800) roll requested 2024-11-13T01:31:02,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462807 2024-11-13T01:31:02,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462807 2024-11-13T01:31:02,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,815 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462807) roll requested 2024-11-13T01:31:02,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462815 2024-11-13T01:31:02,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462815 2024-11-13T01:31:02,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462815) roll requested 2024-11-13T01:31:02,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462824 2024-11-13T01:31:02,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462824 2024-11-13T01:31:02,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462824) roll requested 2024-11-13T01:31:02,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462830 2024-11-13T01:31:02,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462830 2024-11-13T01:31:02,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,837 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462830) roll requested 2024-11-13T01:31:02,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462837 2024-11-13T01:31:02,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462837 2024-11-13T01:31:02,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,846 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462837) roll requested 2024-11-13T01:31:02,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462846 2024-11-13T01:31:02,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462846 2024-11-13T01:31:02,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462846) roll requested 2024-11-13T01:31:02,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462855 2024-11-13T01:31:02,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462846 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462855 2024-11-13T01:31:02,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462855) roll requested 2024-11-13T01:31:02,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462862 2024-11-13T01:31:02,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462862 2024-11-13T01:31:02,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462862) roll requested 2024-11-13T01:31:02,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462871 2024-11-13T01:31:02,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462871 2024-11-13T01:31:02,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462871) roll requested 2024-11-13T01:31:02,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462879 2024-11-13T01:31:02,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462879 2024-11-13T01:31:02,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462879) roll requested 2024-11-13T01:31:02,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462888 2024-11-13T01:31:02,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462888 2024-11-13T01:31:02,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462888) roll requested 2024-11-13T01:31:02,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462895 2024-11-13T01:31:02,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462895 2024-11-13T01:31:02,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462895) roll requested 2024-11-13T01:31:02,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462901 2024-11-13T01:31:02,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462901 2024-11-13T01:31:02,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462901) roll requested 2024-11-13T01:31:02,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462907 2024-11-13T01:31:02,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462907 2024-11-13T01:31:02,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462907) roll requested 2024-11-13T01:31:02,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462914 2024-11-13T01:31:02,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462907 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462914 2024-11-13T01:31:02,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462914) roll requested 2024-11-13T01:31:02,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462921 2024-11-13T01:31:02,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462921 2024-11-13T01:31:02,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462921) roll requested 2024-11-13T01:31:02,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462927 2024-11-13T01:31:02,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462927 2024-11-13T01:31:02,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462927) roll requested 2024-11-13T01:31:02,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462934 2024-11-13T01:31:02,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462934 2024-11-13T01:31:02,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462934) roll requested 2024-11-13T01:31:02,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462943 2024-11-13T01:31:02,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462943 2024-11-13T01:31:02,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462943) roll requested 2024-11-13T01:31:02,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462948 2024-11-13T01:31:02,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462948 2024-11-13T01:31:02,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462948) roll requested 2024-11-13T01:31:02,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462953 2024-11-13T01:31:02,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462953 2024-11-13T01:31:02,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462953) roll requested 2024-11-13T01:31:02,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462959 2024-11-13T01:31:02,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462959 2024-11-13T01:31:02,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462959) roll requested 2024-11-13T01:31:02,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462966 2024-11-13T01:31:02,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462966 2024-11-13T01:31:02,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,974 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462966) roll requested 2024-11-13T01:31:02,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462974 2024-11-13T01:31:02,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462974 2024-11-13T01:31:02,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462974) roll requested 2024-11-13T01:31:02,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462983 2024-11-13T01:31:02,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:02,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462974 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462983 2024-11-13T01:31:02,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:02,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:02,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462983) roll requested 2024-11-13T01:31:02,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461462992 2024-11-13T01:31:03,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462992 2024-11-13T01:31:03,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461462992) roll requested 2024-11-13T01:31:03,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463001 2024-11-13T01:31:03,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461462992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463001 2024-11-13T01:31:03,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463001) roll requested 2024-11-13T01:31:03,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463009 2024-11-13T01:31:03,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463009 2024-11-13T01:31:03,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463009) roll requested 2024-11-13T01:31:03,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463018 2024-11-13T01:31:03,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463018 2024-11-13T01:31:03,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463018) roll requested 2024-11-13T01:31:03,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463025 2024-11-13T01:31:03,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463025 2024-11-13T01:31:03,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463025) roll requested 2024-11-13T01:31:03,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463034 2024-11-13T01:31:03,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463034 2024-11-13T01:31:03,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463034) roll requested 2024-11-13T01:31:03,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463043 2024-11-13T01:31:03,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463043 2024-11-13T01:31:03,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463043) roll requested 2024-11-13T01:31:03,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463052 2024-11-13T01:31:03,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463052 2024-11-13T01:31:03,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463052) roll requested 2024-11-13T01:31:03,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463059 2024-11-13T01:31:03,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463052 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463059 2024-11-13T01:31:03,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463059) roll requested 2024-11-13T01:31:03,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463065 2024-11-13T01:31:03,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,070 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463065 2024-11-13T01:31:03,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463065) roll requested 2024-11-13T01:31:03,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463072 2024-11-13T01:31:03,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463072 2024-11-13T01:31:03,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463072) roll requested 2024-11-13T01:31:03,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463078 2024-11-13T01:31:03,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463078 2024-11-13T01:31:03,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463078) roll requested 2024-11-13T01:31:03,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463085 2024-11-13T01:31:03,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463085 2024-11-13T01:31:03,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463085) roll requested 2024-11-13T01:31:03,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463090 2024-11-13T01:31:03,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463090 2024-11-13T01:31:03,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463090) roll requested 2024-11-13T01:31:03,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463095 2024-11-13T01:31:03,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463095 2024-11-13T01:31:03,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463095) roll requested 2024-11-13T01:31:03,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463101 2024-11-13T01:31:03,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463101 2024-11-13T01:31:03,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463101) roll requested 2024-11-13T01:31:03,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463107 2024-11-13T01:31:03,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463107 2024-11-13T01:31:03,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463107) roll requested 2024-11-13T01:31:03,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463114 2024-11-13T01:31:03,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463114 2024-11-13T01:31:03,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463114) roll requested 2024-11-13T01:31:03,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463120 2024-11-13T01:31:03,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463120 2024-11-13T01:31:03,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463120) roll requested 2024-11-13T01:31:03,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463125 2024-11-13T01:31:03,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463125 2024-11-13T01:31:03,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463125) roll requested 2024-11-13T01:31:03,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463130 2024-11-13T01:31:03,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463130 2024-11-13T01:31:03,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463130) roll requested 2024-11-13T01:31:03,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463135 2024-11-13T01:31:03,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463135 2024-11-13T01:31:03,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463135) roll requested 2024-11-13T01:31:03,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463141 2024-11-13T01:31:03,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463141 2024-11-13T01:31:03,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463141) roll requested 2024-11-13T01:31:03,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463146 2024-11-13T01:31:03,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463146 2024-11-13T01:31:03,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463146) roll requested 2024-11-13T01:31:03,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463150 2024-11-13T01:31:03,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463150 2024-11-13T01:31:03,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463150) roll requested 2024-11-13T01:31:03,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463157 2024-11-13T01:31:03,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463157 2024-11-13T01:31:03,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463157) roll requested 2024-11-13T01:31:03,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463164 2024-11-13T01:31:03,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463164 2024-11-13T01:31:03,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463164) roll requested 2024-11-13T01:31:03,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463172 2024-11-13T01:31:03,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463172 2024-11-13T01:31:03,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463172) roll requested 2024-11-13T01:31:03,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463178 2024-11-13T01:31:03,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463178 2024-11-13T01:31:03,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463178) roll requested 2024-11-13T01:31:03,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463184 2024-11-13T01:31:03,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463184 2024-11-13T01:31:03,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463184) roll requested 2024-11-13T01:31:03,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463189 2024-11-13T01:31:03,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463189 2024-11-13T01:31:03,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,194 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463189) roll requested 2024-11-13T01:31:03,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463195 2024-11-13T01:31:03,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463195 2024-11-13T01:31:03,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463195) roll requested 2024-11-13T01:31:03,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463200 2024-11-13T01:31:03,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463200 2024-11-13T01:31:03,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463200) roll requested 2024-11-13T01:31:03,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463205 2024-11-13T01:31:03,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463205 2024-11-13T01:31:03,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463205) roll requested 2024-11-13T01:31:03,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463210 2024-11-13T01:31:03,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463210 2024-11-13T01:31:03,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463210) roll requested 2024-11-13T01:31:03,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463215 2024-11-13T01:31:03,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463215 2024-11-13T01:31:03,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463215) roll requested 2024-11-13T01:31:03,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463220 2024-11-13T01:31:03,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463215 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463220 2024-11-13T01:31:03,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,225 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463220) roll requested 2024-11-13T01:31:03,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463225 2024-11-13T01:31:03,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463225 2024-11-13T01:31:03,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463225) roll requested 2024-11-13T01:31:03,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463230 2024-11-13T01:31:03,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463230 2024-11-13T01:31:03,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463230) roll requested 2024-11-13T01:31:03,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463235 2024-11-13T01:31:03,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463235 2024-11-13T01:31:03,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463235) roll requested 2024-11-13T01:31:03,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463241 2024-11-13T01:31:03,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463241 2024-11-13T01:31:03,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463241) roll requested 2024-11-13T01:31:03,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463246 2024-11-13T01:31:03,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463246 2024-11-13T01:31:03,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463246) roll requested 2024-11-13T01:31:03,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463252 2024-11-13T01:31:03,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463252 2024-11-13T01:31:03,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463252) roll requested 2024-11-13T01:31:03,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463257 2024-11-13T01:31:03,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463257 2024-11-13T01:31:03,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463257) roll requested 2024-11-13T01:31:03,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463263 2024-11-13T01:31:03,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463263 2024-11-13T01:31:03,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463263) roll requested 2024-11-13T01:31:03,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463268 2024-11-13T01:31:03,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,272 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463268 2024-11-13T01:31:03,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,273 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463268) roll requested 2024-11-13T01:31:03,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463273 2024-11-13T01:31:03,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463273 2024-11-13T01:31:03,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,278 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463273) roll requested 2024-11-13T01:31:03,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463279 2024-11-13T01:31:03,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463273 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463279 2024-11-13T01:31:03,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463279) roll requested 2024-11-13T01:31:03,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463284 2024-11-13T01:31:03,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463284 2024-11-13T01:31:03,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,289 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463284) roll requested 2024-11-13T01:31:03,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463289 2024-11-13T01:31:03,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463289 2024-11-13T01:31:03,294 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,294 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463289) roll requested 2024-11-13T01:31:03,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463295 2024-11-13T01:31:03,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463289 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463295 2024-11-13T01:31:03,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463295) roll requested 2024-11-13T01:31:03,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463300 2024-11-13T01:31:03,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463295 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463300 2024-11-13T01:31:03,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463300) roll requested 2024-11-13T01:31:03,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463305 2024-11-13T01:31:03,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463305 2024-11-13T01:31:03,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463305) roll requested 2024-11-13T01:31:03,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463313 2024-11-13T01:31:03,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463313 2024-11-13T01:31:03,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463313) roll requested 2024-11-13T01:31:03,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463320 2024-11-13T01:31:03,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463320 2024-11-13T01:31:03,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463320) roll requested 2024-11-13T01:31:03,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463328 2024-11-13T01:31:03,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463328 2024-11-13T01:31:03,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463328) roll requested 2024-11-13T01:31:03,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463336 2024-11-13T01:31:03,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463336 2024-11-13T01:31:03,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,343 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463336) roll requested 2024-11-13T01:31:03,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463343 2024-11-13T01:31:03,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463343 2024-11-13T01:31:03,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463343) roll requested 2024-11-13T01:31:03,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463351 2024-11-13T01:31:03,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463351 2024-11-13T01:31:03,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463351) roll requested 2024-11-13T01:31:03,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463360 2024-11-13T01:31:03,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463360 2024-11-13T01:31:03,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463360) roll requested 2024-11-13T01:31:03,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463369 2024-11-13T01:31:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463369 2024-11-13T01:31:03,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463369) roll requested 2024-11-13T01:31:03,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463379 2024-11-13T01:31:03,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463379 2024-11-13T01:31:03,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463379) roll requested 2024-11-13T01:31:03,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463387 2024-11-13T01:31:03,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463387 2024-11-13T01:31:03,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463387) roll requested 2024-11-13T01:31:03,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463396 2024-11-13T01:31:03,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463396 2024-11-13T01:31:03,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463396) roll requested 2024-11-13T01:31:03,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463405 2024-11-13T01:31:03,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463405 2024-11-13T01:31:03,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463405) roll requested 2024-11-13T01:31:03,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463413 2024-11-13T01:31:03,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463405 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463413 2024-11-13T01:31:03,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463413) roll requested 2024-11-13T01:31:03,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463423 2024-11-13T01:31:03,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463423 2024-11-13T01:31:03,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463423) roll requested 2024-11-13T01:31:03,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463434 2024-11-13T01:31:03,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463434 2024-11-13T01:31:03,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463434) roll requested 2024-11-13T01:31:03,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463443 2024-11-13T01:31:03,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463443 2024-11-13T01:31:03,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463443) roll requested 2024-11-13T01:31:03,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463450 2024-11-13T01:31:03,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463450 2024-11-13T01:31:03,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463450) roll requested 2024-11-13T01:31:03,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463458 2024-11-13T01:31:03,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463458 2024-11-13T01:31:03,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463458) roll requested 2024-11-13T01:31:03,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463467 2024-11-13T01:31:03,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463467 2024-11-13T01:31:03,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463467) roll requested 2024-11-13T01:31:03,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463476 2024-11-13T01:31:03,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463476 2024-11-13T01:31:03,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463476) roll requested 2024-11-13T01:31:03,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463485 2024-11-13T01:31:03,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463485 2024-11-13T01:31:03,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463485) roll requested 2024-11-13T01:31:03,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463492 2024-11-13T01:31:03,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463492 2024-11-13T01:31:03,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463492) roll requested 2024-11-13T01:31:03,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463502 2024-11-13T01:31:03,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463502 2024-11-13T01:31:03,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463502) roll requested 2024-11-13T01:31:03,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463509 2024-11-13T01:31:03,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463509 2024-11-13T01:31:03,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463509) roll requested 2024-11-13T01:31:03,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463514 2024-11-13T01:31:03,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463514 2024-11-13T01:31:03,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463514) roll requested 2024-11-13T01:31:03,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463519 2024-11-13T01:31:03,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463519 2024-11-13T01:31:03,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463519) roll requested 2024-11-13T01:31:03,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463528 2024-11-13T01:31:03,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463528 2024-11-13T01:31:03,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463528) roll requested 2024-11-13T01:31:03,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463536 2024-11-13T01:31:03,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,542 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,542 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,542 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,542 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463536 2024-11-13T01:31:03,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463536) roll requested 2024-11-13T01:31:03,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463543 2024-11-13T01:31:03,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463543 2024-11-13T01:31:03,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463543) roll requested 2024-11-13T01:31:03,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463548 2024-11-13T01:31:03,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463543 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463548 2024-11-13T01:31:03,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463548) roll requested 2024-11-13T01:31:03,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463552 2024-11-13T01:31:03,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463552 2024-11-13T01:31:03,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463552) roll requested 2024-11-13T01:31:03,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463557 2024-11-13T01:31:03,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463557 2024-11-13T01:31:03,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463557) roll requested 2024-11-13T01:31:03,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463563 2024-11-13T01:31:03,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463557 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463563 2024-11-13T01:31:03,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463563) roll requested 2024-11-13T01:31:03,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463570 2024-11-13T01:31:03,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463570 2024-11-13T01:31:03,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463570) roll requested 2024-11-13T01:31:03,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463577 2024-11-13T01:31:03,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463577 2024-11-13T01:31:03,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463577) roll requested 2024-11-13T01:31:03,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463586 2024-11-13T01:31:03,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463586 2024-11-13T01:31:03,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463586) roll requested 2024-11-13T01:31:03,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463597 2024-11-13T01:31:03,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463597 2024-11-13T01:31:03,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463597) roll requested 2024-11-13T01:31:03,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463604 2024-11-13T01:31:03,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463604 2024-11-13T01:31:03,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463604) roll requested 2024-11-13T01:31:03,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463612 2024-11-13T01:31:03,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463612 2024-11-13T01:31:03,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463612) roll requested 2024-11-13T01:31:03,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463620 2024-11-13T01:31:03,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463620 2024-11-13T01:31:03,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463620) roll requested 2024-11-13T01:31:03,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463629 2024-11-13T01:31:03,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463629 2024-11-13T01:31:03,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463629) roll requested 2024-11-13T01:31:03,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463638 2024-11-13T01:31:03,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463638 2024-11-13T01:31:03,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463638) roll requested 2024-11-13T01:31:03,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463645 2024-11-13T01:31:03,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463645 2024-11-13T01:31:03,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463645) roll requested 2024-11-13T01:31:03,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463654 2024-11-13T01:31:03,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463654 2024-11-13T01:31:03,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463654) roll requested 2024-11-13T01:31:03,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463664 2024-11-13T01:31:03,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463664 2024-11-13T01:31:03,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463664) roll requested 2024-11-13T01:31:03,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463673 2024-11-13T01:31:03,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463673 2024-11-13T01:31:03,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463673) roll requested 2024-11-13T01:31:03,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463684 2024-11-13T01:31:03,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463684 2024-11-13T01:31:03,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463684) roll requested 2024-11-13T01:31:03,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463694 2024-11-13T01:31:03,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463694 2024-11-13T01:31:03,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463694) roll requested 2024-11-13T01:31:03,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463704 2024-11-13T01:31:03,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463704 2024-11-13T01:31:03,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463704) roll requested 2024-11-13T01:31:03,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463709 2024-11-13T01:31:03,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463709 2024-11-13T01:31:03,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463709) roll requested 2024-11-13T01:31:03,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463714 2024-11-13T01:31:03,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463714 2024-11-13T01:31:03,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463714) roll requested 2024-11-13T01:31:03,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463719 2024-11-13T01:31:03,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463719 2024-11-13T01:31:03,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463719) roll requested 2024-11-13T01:31:03,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463725 2024-11-13T01:31:03,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463725 2024-11-13T01:31:03,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463725) roll requested 2024-11-13T01:31:03,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463731 2024-11-13T01:31:03,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463731 2024-11-13T01:31:03,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463731) roll requested 2024-11-13T01:31:03,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463737 2024-11-13T01:31:03,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463737 2024-11-13T01:31:03,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463737) roll requested 2024-11-13T01:31:03,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463744 2024-11-13T01:31:03,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463744 2024-11-13T01:31:03,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463744) roll requested 2024-11-13T01:31:03,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463752 2024-11-13T01:31:03,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463752 2024-11-13T01:31:03,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463752) roll requested 2024-11-13T01:31:03,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463760 2024-11-13T01:31:03,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463760 2024-11-13T01:31:03,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463760) roll requested 2024-11-13T01:31:03,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463768 2024-11-13T01:31:03,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463768 2024-11-13T01:31:03,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463768) roll requested 2024-11-13T01:31:03,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463774 2024-11-13T01:31:03,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463774 2024-11-13T01:31:03,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463774) roll requested 2024-11-13T01:31:03,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463780 2024-11-13T01:31:03,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463780 2024-11-13T01:31:03,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463780) roll requested 2024-11-13T01:31:03,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463787 2024-11-13T01:31:03,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463780 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463787 2024-11-13T01:31:03,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463787) roll requested 2024-11-13T01:31:03,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463795 2024-11-13T01:31:03,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463795 2024-11-13T01:31:03,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463795) roll requested 2024-11-13T01:31:03,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463803 2024-11-13T01:31:03,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463803 2024-11-13T01:31:03,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463803) roll requested 2024-11-13T01:31:03,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463812 2024-11-13T01:31:03,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463812 2024-11-13T01:31:03,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463812) roll requested 2024-11-13T01:31:03,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463822 2024-11-13T01:31:03,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463822 2024-11-13T01:31:03,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,830 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,830 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463822) roll requested 2024-11-13T01:31:03,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463830 2024-11-13T01:31:03,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463830 2024-11-13T01:31:03,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463830) roll requested 2024-11-13T01:31:03,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463840 2024-11-13T01:31:03,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463830 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463840 2024-11-13T01:31:03,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463840) roll requested 2024-11-13T01:31:03,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463848 2024-11-13T01:31:03,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463848 2024-11-13T01:31:03,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463848) roll requested 2024-11-13T01:31:03,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463857 2024-11-13T01:31:03,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463857 2024-11-13T01:31:03,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,865 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463857) roll requested 2024-11-13T01:31:03,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463865 2024-11-13T01:31:03,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,870 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463865 2024-11-13T01:31:03,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463865) roll requested 2024-11-13T01:31:03,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463870 2024-11-13T01:31:03,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463865 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463870 2024-11-13T01:31:03,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463870) roll requested 2024-11-13T01:31:03,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463877 2024-11-13T01:31:03,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463877 2024-11-13T01:31:03,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463877) roll requested 2024-11-13T01:31:03,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463885 2024-11-13T01:31:03,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463877 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463885 2024-11-13T01:31:03,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463885) roll requested 2024-11-13T01:31:03,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463894 2024-11-13T01:31:03,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463894 2024-11-13T01:31:03,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463894) roll requested 2024-11-13T01:31:03,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463904 2024-11-13T01:31:03,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463904 2024-11-13T01:31:03,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,913 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463904) roll requested 2024-11-13T01:31:03,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463913 2024-11-13T01:31:03,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463913 2024-11-13T01:31:03,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463913) roll requested 2024-11-13T01:31:03,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463919 2024-11-13T01:31:03,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463919 2024-11-13T01:31:03,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463919) roll requested 2024-11-13T01:31:03,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463925 2024-11-13T01:31:03,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463925 2024-11-13T01:31:03,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463925) roll requested 2024-11-13T01:31:03,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463933 2024-11-13T01:31:03,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463933 2024-11-13T01:31:03,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463933) roll requested 2024-11-13T01:31:03,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463941 2024-11-13T01:31:03,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463941 2024-11-13T01:31:03,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463941) roll requested 2024-11-13T01:31:03,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463949 2024-11-13T01:31:03,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463949 2024-11-13T01:31:03,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,957 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463949) roll requested 2024-11-13T01:31:03,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463957 2024-11-13T01:31:03,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463957 2024-11-13T01:31:03,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,965 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463957) roll requested 2024-11-13T01:31:03,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463965 2024-11-13T01:31:03,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463965 2024-11-13T01:31:03,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463965) roll requested 2024-11-13T01:31:03,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463973 2024-11-13T01:31:03,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463973 2024-11-13T01:31:03,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,980 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463973) roll requested 2024-11-13T01:31:03,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463980 2024-11-13T01:31:03,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463980 2024-11-13T01:31:03,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463980) roll requested 2024-11-13T01:31:03,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463989 2024-11-13T01:31:03,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:03,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463980 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463989 2024-11-13T01:31:03,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:03,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:03,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463989) roll requested 2024-11-13T01:31:03,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461463998 2024-11-13T01:31:04,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463998 2024-11-13T01:31:04,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461463998) roll requested 2024-11-13T01:31:04,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464005 2024-11-13T01:31:04,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461463998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464005 2024-11-13T01:31:04,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464005) roll requested 2024-11-13T01:31:04,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464013 2024-11-13T01:31:04,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,021 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464013 2024-11-13T01:31:04,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464013) roll requested 2024-11-13T01:31:04,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464022 2024-11-13T01:31:04,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464022 2024-11-13T01:31:04,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464022) roll requested 2024-11-13T01:31:04,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464031 2024-11-13T01:31:04,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464022 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464031 2024-11-13T01:31:04,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464031) roll requested 2024-11-13T01:31:04,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464041 2024-11-13T01:31:04,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464041 2024-11-13T01:31:04,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464041) roll requested 2024-11-13T01:31:04,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464049 2024-11-13T01:31:04,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464049 2024-11-13T01:31:04,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464049) roll requested 2024-11-13T01:31:04,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464056 2024-11-13T01:31:04,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464056 2024-11-13T01:31:04,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464056) roll requested 2024-11-13T01:31:04,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464063 2024-11-13T01:31:04,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464063 2024-11-13T01:31:04,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464063) roll requested 2024-11-13T01:31:04,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464069 2024-11-13T01:31:04,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464069 2024-11-13T01:31:04,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464069) roll requested 2024-11-13T01:31:04,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464075 2024-11-13T01:31:04,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464075 2024-11-13T01:31:04,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464075) roll requested 2024-11-13T01:31:04,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464082 2024-11-13T01:31:04,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464075 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464082 2024-11-13T01:31:04,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464082) roll requested 2024-11-13T01:31:04,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464090 2024-11-13T01:31:04,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464090 2024-11-13T01:31:04,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,097 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464090) roll requested 2024-11-13T01:31:04,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464097 2024-11-13T01:31:04,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464097 2024-11-13T01:31:04,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464097) roll requested 2024-11-13T01:31:04,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464106 2024-11-13T01:31:04,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464106 2024-11-13T01:31:04,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464106) roll requested 2024-11-13T01:31:04,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464116 2024-11-13T01:31:04,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464116 2024-11-13T01:31:04,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464116) roll requested 2024-11-13T01:31:04,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464124 2024-11-13T01:31:04,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464124 2024-11-13T01:31:04,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464124) roll requested 2024-11-13T01:31:04,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464132 2024-11-13T01:31:04,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464132 2024-11-13T01:31:04,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464132) roll requested 2024-11-13T01:31:04,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464139 2024-11-13T01:31:04,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464132 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464139 2024-11-13T01:31:04,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464139) roll requested 2024-11-13T01:31:04,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464147 2024-11-13T01:31:04,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464147 2024-11-13T01:31:04,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,155 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464147) roll requested 2024-11-13T01:31:04,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464155 2024-11-13T01:31:04,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464147 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464155 2024-11-13T01:31:04,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464155) roll requested 2024-11-13T01:31:04,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464162 2024-11-13T01:31:04,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464155 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464162 2024-11-13T01:31:04,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464162) roll requested 2024-11-13T01:31:04,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464170 2024-11-13T01:31:04,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464170 2024-11-13T01:31:04,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,178 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464170) roll requested 2024-11-13T01:31:04,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464178 2024-11-13T01:31:04,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464178 2024-11-13T01:31:04,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464178) roll requested 2024-11-13T01:31:04,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464188 2024-11-13T01:31:04,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464178 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464188 2024-11-13T01:31:04,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464188) roll requested 2024-11-13T01:31:04,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464198 2024-11-13T01:31:04,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464198 2024-11-13T01:31:04,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464198) roll requested 2024-11-13T01:31:04,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464207 2024-11-13T01:31:04,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464207 2024-11-13T01:31:04,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,214 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464207) roll requested 2024-11-13T01:31:04,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464214 2024-11-13T01:31:04,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464207 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464214 2024-11-13T01:31:04,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464214) roll requested 2024-11-13T01:31:04,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464221 2024-11-13T01:31:04,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464221 2024-11-13T01:31:04,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,228 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464221) roll requested 2024-11-13T01:31:04,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464228 2024-11-13T01:31:04,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464228 2024-11-13T01:31:04,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464228) roll requested 2024-11-13T01:31:04,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464237 2024-11-13T01:31:04,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464237 2024-11-13T01:31:04,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464237) roll requested 2024-11-13T01:31:04,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464245 2024-11-13T01:31:04,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464245 2024-11-13T01:31:04,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464245) roll requested 2024-11-13T01:31:04,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464252 2024-11-13T01:31:04,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464252 2024-11-13T01:31:04,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464252) roll requested 2024-11-13T01:31:04,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464263 2024-11-13T01:31:04,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464263 2024-11-13T01:31:04,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,269 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464263) roll requested 2024-11-13T01:31:04,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464269 2024-11-13T01:31:04,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464269 2024-11-13T01:31:04,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464269) roll requested 2024-11-13T01:31:04,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464275 2024-11-13T01:31:04,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464275 2024-11-13T01:31:04,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464275) roll requested 2024-11-13T01:31:04,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464283 2024-11-13T01:31:04,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464283 2024-11-13T01:31:04,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464283) roll requested 2024-11-13T01:31:04,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464293 2024-11-13T01:31:04,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464283 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464293 2024-11-13T01:31:04,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,302 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464293) roll requested 2024-11-13T01:31:04,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464303 2024-11-13T01:31:04,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464303 2024-11-13T01:31:04,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464303) roll requested 2024-11-13T01:31:04,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464312 2024-11-13T01:31:04,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464303 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464312 2024-11-13T01:31:04,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464312) roll requested 2024-11-13T01:31:04,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464320 2024-11-13T01:31:04,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464320 2024-11-13T01:31:04,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,329 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464320) roll requested 2024-11-13T01:31:04,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464329 2024-11-13T01:31:04,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464329 2024-11-13T01:31:04,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464329) roll requested 2024-11-13T01:31:04,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464335 2024-11-13T01:31:04,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464329 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464335 2024-11-13T01:31:04,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464335) roll requested 2024-11-13T01:31:04,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464342 2024-11-13T01:31:04,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464342 2024-11-13T01:31:04,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464342) roll requested 2024-11-13T01:31:04,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464347 2024-11-13T01:31:04,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464347 2024-11-13T01:31:04,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464347) roll requested 2024-11-13T01:31:04,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464352 2024-11-13T01:31:04,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464352 2024-11-13T01:31:04,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464352) roll requested 2024-11-13T01:31:04,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464359 2024-11-13T01:31:04,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464359 2024-11-13T01:31:04,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464359) roll requested 2024-11-13T01:31:04,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464367 2024-11-13T01:31:04,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464367 2024-11-13T01:31:04,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464367) roll requested 2024-11-13T01:31:04,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464373 2024-11-13T01:31:04,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,426 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,426 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464373 2024-11-13T01:31:04,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464373) roll requested 2024-11-13T01:31:04,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464428 2024-11-13T01:31:04,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464428 2024-11-13T01:31:04,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464428) roll requested 2024-11-13T01:31:04,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464437 2024-11-13T01:31:04,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464437 2024-11-13T01:31:04,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464437) roll requested 2024-11-13T01:31:04,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464446 2024-11-13T01:31:04,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464446 2024-11-13T01:31:04,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464446) roll requested 2024-11-13T01:31:04,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464452 2024-11-13T01:31:04,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464452 2024-11-13T01:31:04,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464452) roll requested 2024-11-13T01:31:04,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464458 2024-11-13T01:31:04,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464458 2024-11-13T01:31:04,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464458) roll requested 2024-11-13T01:31:04,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464466 2024-11-13T01:31:04,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464466 2024-11-13T01:31:04,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464466) roll requested 2024-11-13T01:31:04,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464476 2024-11-13T01:31:04,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464476 2024-11-13T01:31:04,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,483 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464476) roll requested 2024-11-13T01:31:04,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464484 2024-11-13T01:31:04,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464484 2024-11-13T01:31:04,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464484) roll requested 2024-11-13T01:31:04,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464490 2024-11-13T01:31:04,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464490 2024-11-13T01:31:04,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464490) roll requested 2024-11-13T01:31:04,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464498 2024-11-13T01:31:04,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464498 2024-11-13T01:31:04,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464498) roll requested 2024-11-13T01:31:04,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464507 2024-11-13T01:31:04,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464507 2024-11-13T01:31:04,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464507) roll requested 2024-11-13T01:31:04,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464515 2024-11-13T01:31:04,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464507 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464515 2024-11-13T01:31:04,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464515) roll requested 2024-11-13T01:31:04,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464522 2024-11-13T01:31:04,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464522 2024-11-13T01:31:04,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464522) roll requested 2024-11-13T01:31:04,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464530 2024-11-13T01:31:04,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464530 2024-11-13T01:31:04,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464530) roll requested 2024-11-13T01:31:04,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464537 2024-11-13T01:31:04,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464537 2024-11-13T01:31:04,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,548 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464537) roll requested 2024-11-13T01:31:04,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464548 2024-11-13T01:31:04,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464548 2024-11-13T01:31:04,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464548) roll requested 2024-11-13T01:31:04,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464554 2024-11-13T01:31:04,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464554 2024-11-13T01:31:04,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464554) roll requested 2024-11-13T01:31:04,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464559 2024-11-13T01:31:04,564 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,564 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,564 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,564 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464559 2024-11-13T01:31:04,565 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,565 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,565 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464559) roll requested 2024-11-13T01:31:04,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464565 2024-11-13T01:31:04,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464565 2024-11-13T01:31:04,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464565) roll requested 2024-11-13T01:31:04,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464572 2024-11-13T01:31:04,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464572 2024-11-13T01:31:04,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464572) roll requested 2024-11-13T01:31:04,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464579 2024-11-13T01:31:04,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464579 2024-11-13T01:31:04,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464579) roll requested 2024-11-13T01:31:04,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464588 2024-11-13T01:31:04,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464579 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464588 2024-11-13T01:31:04,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464588) roll requested 2024-11-13T01:31:04,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464598 2024-11-13T01:31:04,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464598 2024-11-13T01:31:04,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464598) roll requested 2024-11-13T01:31:04,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464608 2024-11-13T01:31:04,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464608 2024-11-13T01:31:04,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464608) roll requested 2024-11-13T01:31:04,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464618 2024-11-13T01:31:04,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464618 2024-11-13T01:31:04,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464618) roll requested 2024-11-13T01:31:04,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464624 2024-11-13T01:31:04,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464618 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464624 2024-11-13T01:31:04,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,630 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464624) roll requested 2024-11-13T01:31:04,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464631 2024-11-13T01:31:04,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464631 2024-11-13T01:31:04,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464631) roll requested 2024-11-13T01:31:04,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464639 2024-11-13T01:31:04,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464639 2024-11-13T01:31:04,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464639) roll requested 2024-11-13T01:31:04,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464648 2024-11-13T01:31:04,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464648 2024-11-13T01:31:04,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464648) roll requested 2024-11-13T01:31:04,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464657 2024-11-13T01:31:04,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464657 2024-11-13T01:31:04,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,664 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464657) roll requested 2024-11-13T01:31:04,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464664 2024-11-13T01:31:04,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464664 2024-11-13T01:31:04,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464664) roll requested 2024-11-13T01:31:04,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464673 2024-11-13T01:31:04,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464664 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464673 2024-11-13T01:31:04,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464673) roll requested 2024-11-13T01:31:04,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464682 2024-11-13T01:31:04,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464673 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464682 2024-11-13T01:31:04,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464682) roll requested 2024-11-13T01:31:04,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464691 2024-11-13T01:31:04,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464691 2024-11-13T01:31:04,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464691) roll requested 2024-11-13T01:31:04,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464700 2024-11-13T01:31:04,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464700 2024-11-13T01:31:04,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464700) roll requested 2024-11-13T01:31:04,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464710 2024-11-13T01:31:04,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464710 2024-11-13T01:31:04,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464710) roll requested 2024-11-13T01:31:04,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464719 2024-11-13T01:31:04,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464719 2024-11-13T01:31:04,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464719) roll requested 2024-11-13T01:31:04,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464726 2024-11-13T01:31:04,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464726 2024-11-13T01:31:04,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464726) roll requested 2024-11-13T01:31:04,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464733 2024-11-13T01:31:04,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464733 2024-11-13T01:31:04,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464733) roll requested 2024-11-13T01:31:04,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464740 2024-11-13T01:31:04,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464740 2024-11-13T01:31:04,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464740) roll requested 2024-11-13T01:31:04,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464749 2024-11-13T01:31:04,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464749 2024-11-13T01:31:04,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464749) roll requested 2024-11-13T01:31:04,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464757 2024-11-13T01:31:04,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464757 2024-11-13T01:31:04,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464757) roll requested 2024-11-13T01:31:04,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464764 2024-11-13T01:31:04,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464757 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464764 2024-11-13T01:31:04,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464764) roll requested 2024-11-13T01:31:04,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464771 2024-11-13T01:31:04,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464771 2024-11-13T01:31:04,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464771) roll requested 2024-11-13T01:31:04,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464779 2024-11-13T01:31:04,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464779 2024-11-13T01:31:04,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464779) roll requested 2024-11-13T01:31:04,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464786 2024-11-13T01:31:04,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464786 2024-11-13T01:31:04,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464786) roll requested 2024-11-13T01:31:04,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464796 2024-11-13T01:31:04,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464796 2024-11-13T01:31:04,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464796) roll requested 2024-11-13T01:31:04,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464803 2024-11-13T01:31:04,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464803 2024-11-13T01:31:04,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464803) roll requested 2024-11-13T01:31:04,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464807 2024-11-13T01:31:04,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464807 2024-11-13T01:31:04,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464807) roll requested 2024-11-13T01:31:04,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464812 2024-11-13T01:31:04,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464812 2024-11-13T01:31:04,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464812) roll requested 2024-11-13T01:31:04,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464819 2024-11-13T01:31:04,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464819 2024-11-13T01:31:04,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464819) roll requested 2024-11-13T01:31:04,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464825 2024-11-13T01:31:04,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464825 2024-11-13T01:31:04,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464825) roll requested 2024-11-13T01:31:04,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464833 2024-11-13T01:31:04,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464833 2024-11-13T01:31:04,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464833) roll requested 2024-11-13T01:31:04,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464843 2024-11-13T01:31:04,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464843 2024-11-13T01:31:04,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464843) roll requested 2024-11-13T01:31:04,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464852 2024-11-13T01:31:04,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464852 2024-11-13T01:31:04,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464852) roll requested 2024-11-13T01:31:04,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464861 2024-11-13T01:31:04,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464861 2024-11-13T01:31:04,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464861) roll requested 2024-11-13T01:31:04,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464871 2024-11-13T01:31:04,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464871 2024-11-13T01:31:04,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464871) roll requested 2024-11-13T01:31:04,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464880 2024-11-13T01:31:04,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464880 2024-11-13T01:31:04,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464880) roll requested 2024-11-13T01:31:04,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464887 2024-11-13T01:31:04,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464887 2024-11-13T01:31:04,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464887) roll requested 2024-11-13T01:31:04,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464894 2024-11-13T01:31:04,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464894 2024-11-13T01:31:04,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464894) roll requested 2024-11-13T01:31:04,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464901 2024-11-13T01:31:04,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,908 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464901 2024-11-13T01:31:04,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464901) roll requested 2024-11-13T01:31:04,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464909 2024-11-13T01:31:04,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464909 2024-11-13T01:31:04,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464909) roll requested 2024-11-13T01:31:04,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464917 2024-11-13T01:31:04,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464917 2024-11-13T01:31:04,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464917) roll requested 2024-11-13T01:31:04,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464926 2024-11-13T01:31:04,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464926 2024-11-13T01:31:04,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,933 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464926) roll requested 2024-11-13T01:31:04,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464933 2024-11-13T01:31:04,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464933 2024-11-13T01:31:04,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464933) roll requested 2024-11-13T01:31:04,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464940 2024-11-13T01:31:04,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464940 2024-11-13T01:31:04,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464940) roll requested 2024-11-13T01:31:04,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464948 2024-11-13T01:31:04,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464948 2024-11-13T01:31:04,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464948) roll requested 2024-11-13T01:31:04,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464956 2024-11-13T01:31:04,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464956 2024-11-13T01:31:04,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464956) roll requested 2024-11-13T01:31:04,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464962 2024-11-13T01:31:04,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464962 2024-11-13T01:31:04,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464962) roll requested 2024-11-13T01:31:04,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464968 2024-11-13T01:31:04,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464968 2024-11-13T01:31:04,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464968) roll requested 2024-11-13T01:31:04,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464976 2024-11-13T01:31:04,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464976 2024-11-13T01:31:04,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464976) roll requested 2024-11-13T01:31:04,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464984 2024-11-13T01:31:04,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:04,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464984 2024-11-13T01:31:04,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:04,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:04,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464984) roll requested 2024-11-13T01:31:04,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461464993 2024-11-13T01:31:05,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464993 2024-11-13T01:31:05,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461464993) roll requested 2024-11-13T01:31:05,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465002 2024-11-13T01:31:05,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461464993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465002 2024-11-13T01:31:05,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465002) roll requested 2024-11-13T01:31:05,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465010 2024-11-13T01:31:05,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465010 2024-11-13T01:31:05,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465010) roll requested 2024-11-13T01:31:05,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465020 2024-11-13T01:31:05,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465020 2024-11-13T01:31:05,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465020) roll requested 2024-11-13T01:31:05,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465030 2024-11-13T01:31:05,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465030 2024-11-13T01:31:05,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465030) roll requested 2024-11-13T01:31:05,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465040 2024-11-13T01:31:05,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465040 2024-11-13T01:31:05,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465040) roll requested 2024-11-13T01:31:05,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465050 2024-11-13T01:31:05,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465050 2024-11-13T01:31:05,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465050) roll requested 2024-11-13T01:31:05,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465060 2024-11-13T01:31:05,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465060 2024-11-13T01:31:05,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465060) roll requested 2024-11-13T01:31:05,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465069 2024-11-13T01:31:05,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465069 2024-11-13T01:31:05,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465069) roll requested 2024-11-13T01:31:05,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465078 2024-11-13T01:31:05,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465078 2024-11-13T01:31:05,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465078) roll requested 2024-11-13T01:31:05,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465084 2024-11-13T01:31:05,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465084 2024-11-13T01:31:05,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465084) roll requested 2024-11-13T01:31:05,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465092 2024-11-13T01:31:05,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465092 2024-11-13T01:31:05,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,101 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465092) roll requested 2024-11-13T01:31:05,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465101 2024-11-13T01:31:05,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465092 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465101 2024-11-13T01:31:05,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465101) roll requested 2024-11-13T01:31:05,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465108 2024-11-13T01:31:05,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465108 2024-11-13T01:31:05,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465108) roll requested 2024-11-13T01:31:05,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465115 2024-11-13T01:31:05,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465115 2024-11-13T01:31:05,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465115) roll requested 2024-11-13T01:31:05,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465123 2024-11-13T01:31:05,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465115 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465123 2024-11-13T01:31:05,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,131 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,131 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465123) roll requested 2024-11-13T01:31:05,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465131 2024-11-13T01:31:05,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465131 2024-11-13T01:31:05,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465131) roll requested 2024-11-13T01:31:05,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465140 2024-11-13T01:31:05,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465131 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465140 2024-11-13T01:31:05,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465140) roll requested 2024-11-13T01:31:05,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465150 2024-11-13T01:31:05,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465150 2024-11-13T01:31:05,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,157 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465150) roll requested 2024-11-13T01:31:05,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465157 2024-11-13T01:31:05,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465157 2024-11-13T01:31:05,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465157) roll requested 2024-11-13T01:31:05,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465164 2024-11-13T01:31:05,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465157 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465164 2024-11-13T01:31:05,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465164) roll requested 2024-11-13T01:31:05,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465172 2024-11-13T01:31:05,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465164 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465172 2024-11-13T01:31:05,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,199 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465172) roll requested 2024-11-13T01:31:05,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465199 2024-11-13T01:31:05,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465199 2024-11-13T01:31:05,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465199) roll requested 2024-11-13T01:31:05,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465206 2024-11-13T01:31:05,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465206 2024-11-13T01:31:05,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465206) roll requested 2024-11-13T01:31:05,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465214 2024-11-13T01:31:05,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465214 2024-11-13T01:31:05,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465214) roll requested 2024-11-13T01:31:05,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465223 2024-11-13T01:31:05,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465214 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465223 2024-11-13T01:31:05,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465223) roll requested 2024-11-13T01:31:05,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465231 2024-11-13T01:31:05,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465231 2024-11-13T01:31:05,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465231) roll requested 2024-11-13T01:31:05,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465238 2024-11-13T01:31:05,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465238 2024-11-13T01:31:05,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465238) roll requested 2024-11-13T01:31:05,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465246 2024-11-13T01:31:05,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465246 2024-11-13T01:31:05,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465246) roll requested 2024-11-13T01:31:05,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465255 2024-11-13T01:31:05,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465255 2024-11-13T01:31:05,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465255) roll requested 2024-11-13T01:31:05,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465263 2024-11-13T01:31:05,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465263 2024-11-13T01:31:05,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465263) roll requested 2024-11-13T01:31:05,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465270 2024-11-13T01:31:05,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465270 2024-11-13T01:31:05,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465270) roll requested 2024-11-13T01:31:05,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465277 2024-11-13T01:31:05,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465277 2024-11-13T01:31:05,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465277) roll requested 2024-11-13T01:31:05,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465284 2024-11-13T01:31:05,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465284 2024-11-13T01:31:05,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465284) roll requested 2024-11-13T01:31:05,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465292 2024-11-13T01:31:05,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465292 2024-11-13T01:31:05,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,300 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465292) roll requested 2024-11-13T01:31:05,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465300 2024-11-13T01:31:05,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465300 2024-11-13T01:31:05,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465300) roll requested 2024-11-13T01:31:05,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465308 2024-11-13T01:31:05,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465300 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465308 2024-11-13T01:31:05,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465308) roll requested 2024-11-13T01:31:05,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465313 2024-11-13T01:31:05,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465313 2024-11-13T01:31:05,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465313) roll requested 2024-11-13T01:31:05,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465319 2024-11-13T01:31:05,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465319 2024-11-13T01:31:05,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465319) roll requested 2024-11-13T01:31:05,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465326 2024-11-13T01:31:05,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465326 2024-11-13T01:31:05,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465326) roll requested 2024-11-13T01:31:05,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465335 2024-11-13T01:31:05,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465326 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465335 2024-11-13T01:31:05,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465335) roll requested 2024-11-13T01:31:05,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465344 2024-11-13T01:31:05,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465344 2024-11-13T01:31:05,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465344) roll requested 2024-11-13T01:31:05,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465352 2024-11-13T01:31:05,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465352 2024-11-13T01:31:05,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465352) roll requested 2024-11-13T01:31:05,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465359 2024-11-13T01:31:05,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465359 2024-11-13T01:31:05,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465359) roll requested 2024-11-13T01:31:05,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465368 2024-11-13T01:31:05,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465368 2024-11-13T01:31:05,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465368) roll requested 2024-11-13T01:31:05,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465377 2024-11-13T01:31:05,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465377 2024-11-13T01:31:05,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465377) roll requested 2024-11-13T01:31:05,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465388 2024-11-13T01:31:05,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465388 2024-11-13T01:31:05,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465388) roll requested 2024-11-13T01:31:05,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465400 2024-11-13T01:31:05,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465400 2024-11-13T01:31:05,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465400) roll requested 2024-11-13T01:31:05,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465406 2024-11-13T01:31:05,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465406 2024-11-13T01:31:05,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465406) roll requested 2024-11-13T01:31:05,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465411 2024-11-13T01:31:05,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465411 2024-11-13T01:31:05,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465411) roll requested 2024-11-13T01:31:05,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465416 2024-11-13T01:31:05,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465416 2024-11-13T01:31:05,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465416) roll requested 2024-11-13T01:31:05,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465421 2024-11-13T01:31:05,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465421 2024-11-13T01:31:05,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465421) roll requested 2024-11-13T01:31:05,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465428 2024-11-13T01:31:05,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465428 2024-11-13T01:31:05,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465428) roll requested 2024-11-13T01:31:05,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465434 2024-11-13T01:31:05,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465434 2024-11-13T01:31:05,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465434) roll requested 2024-11-13T01:31:05,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465443 2024-11-13T01:31:05,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465443 2024-11-13T01:31:05,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465443) roll requested 2024-11-13T01:31:05,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465452 2024-11-13T01:31:05,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465452 2024-11-13T01:31:05,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465452) roll requested 2024-11-13T01:31:05,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465458 2024-11-13T01:31:05,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465458 2024-11-13T01:31:05,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465458) roll requested 2024-11-13T01:31:05,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465465 2024-11-13T01:31:05,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465458 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465465 2024-11-13T01:31:05,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465465) roll requested 2024-11-13T01:31:05,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465473 2024-11-13T01:31:05,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465473 2024-11-13T01:31:05,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,482 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465473) roll requested 2024-11-13T01:31:05,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465482 2024-11-13T01:31:05,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465482 2024-11-13T01:31:05,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465482) roll requested 2024-11-13T01:31:05,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465491 2024-11-13T01:31:05,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465482 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465491 2024-11-13T01:31:05,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,498 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465491) roll requested 2024-11-13T01:31:05,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465498 2024-11-13T01:31:05,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465498 2024-11-13T01:31:05,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465498) roll requested 2024-11-13T01:31:05,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465504 2024-11-13T01:31:05,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465504 2024-11-13T01:31:05,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465504) roll requested 2024-11-13T01:31:05,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465511 2024-11-13T01:31:05,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465511 2024-11-13T01:31:05,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465511) roll requested 2024-11-13T01:31:05,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465519 2024-11-13T01:31:05,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465519 2024-11-13T01:31:05,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465519) roll requested 2024-11-13T01:31:05,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465528 2024-11-13T01:31:05,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465519 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465528 2024-11-13T01:31:05,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465528) roll requested 2024-11-13T01:31:05,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465538 2024-11-13T01:31:05,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465528 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465538 2024-11-13T01:31:05,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465538) roll requested 2024-11-13T01:31:05,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465545 2024-11-13T01:31:05,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465545 2024-11-13T01:31:05,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465545) roll requested 2024-11-13T01:31:05,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465552 2024-11-13T01:31:05,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465552 2024-11-13T01:31:05,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465552) roll requested 2024-11-13T01:31:05,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465561 2024-11-13T01:31:05,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465561 2024-11-13T01:31:05,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465561) roll requested 2024-11-13T01:31:05,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465570 2024-11-13T01:31:05,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465570 2024-11-13T01:31:05,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465570) roll requested 2024-11-13T01:31:05,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465580 2024-11-13T01:31:05,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465580 2024-11-13T01:31:05,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465580) roll requested 2024-11-13T01:31:05,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465589 2024-11-13T01:31:05,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465589 2024-11-13T01:31:05,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,598 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465589) roll requested 2024-11-13T01:31:05,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465598 2024-11-13T01:31:05,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465598 2024-11-13T01:31:05,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465598) roll requested 2024-11-13T01:31:05,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465606 2024-11-13T01:31:05,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465606 2024-11-13T01:31:05,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465606) roll requested 2024-11-13T01:31:05,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465612 2024-11-13T01:31:05,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465612 2024-11-13T01:31:05,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465612) roll requested 2024-11-13T01:31:05,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465617 2024-11-13T01:31:05,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465617 2024-11-13T01:31:05,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465617) roll requested 2024-11-13T01:31:05,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465623 2024-11-13T01:31:05,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465623 2024-11-13T01:31:05,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465623) roll requested 2024-11-13T01:31:05,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465629 2024-11-13T01:31:05,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465629 2024-11-13T01:31:05,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465629) roll requested 2024-11-13T01:31:05,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465636 2024-11-13T01:31:05,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465636 2024-11-13T01:31:05,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465636) roll requested 2024-11-13T01:31:05,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465643 2024-11-13T01:31:05,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465643 2024-11-13T01:31:05,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465643) roll requested 2024-11-13T01:31:05,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465650 2024-11-13T01:31:05,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465650 2024-11-13T01:31:05,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465650) roll requested 2024-11-13T01:31:05,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465658 2024-11-13T01:31:05,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465658 2024-11-13T01:31:05,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465658) roll requested 2024-11-13T01:31:05,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465667 2024-11-13T01:31:05,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465667 2024-11-13T01:31:05,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465667) roll requested 2024-11-13T01:31:05,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465675 2024-11-13T01:31:05,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465675 2024-11-13T01:31:05,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465675) roll requested 2024-11-13T01:31:05,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465683 2024-11-13T01:31:05,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465683 2024-11-13T01:31:05,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465683) roll requested 2024-11-13T01:31:05,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465693 2024-11-13T01:31:05,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465693 2024-11-13T01:31:05,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465693) roll requested 2024-11-13T01:31:05,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465700 2024-11-13T01:31:05,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465700 2024-11-13T01:31:05,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465700) roll requested 2024-11-13T01:31:05,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465707 2024-11-13T01:31:05,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465707 2024-11-13T01:31:05,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465707) roll requested 2024-11-13T01:31:05,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465714 2024-11-13T01:31:05,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465714 2024-11-13T01:31:05,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465714) roll requested 2024-11-13T01:31:05,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465722 2024-11-13T01:31:05,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465722 2024-11-13T01:31:05,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465722) roll requested 2024-11-13T01:31:05,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465732 2024-11-13T01:31:05,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465732 2024-11-13T01:31:05,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465732) roll requested 2024-11-13T01:31:05,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465739 2024-11-13T01:31:05,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465739 2024-11-13T01:31:05,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,746 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465739) roll requested 2024-11-13T01:31:05,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465746 2024-11-13T01:31:05,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465746 2024-11-13T01:31:05,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465746) roll requested 2024-11-13T01:31:05,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465753 2024-11-13T01:31:05,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465746 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465753 2024-11-13T01:31:05,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465753) roll requested 2024-11-13T01:31:05,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465760 2024-11-13T01:31:05,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465760 2024-11-13T01:31:05,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465760) roll requested 2024-11-13T01:31:05,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465768 2024-11-13T01:31:05,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465768 2024-11-13T01:31:05,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465768) roll requested 2024-11-13T01:31:05,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465776 2024-11-13T01:31:05,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465776 2024-11-13T01:31:05,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465776) roll requested 2024-11-13T01:31:05,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465784 2024-11-13T01:31:05,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465784 2024-11-13T01:31:05,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465784) roll requested 2024-11-13T01:31:05,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465794 2024-11-13T01:31:05,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465794 2024-11-13T01:31:05,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465794) roll requested 2024-11-13T01:31:05,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465803 2024-11-13T01:31:05,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465794 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465803 2024-11-13T01:31:05,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,810 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465803) roll requested 2024-11-13T01:31:05,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465810 2024-11-13T01:31:05,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465810 2024-11-13T01:31:05,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,817 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465810) roll requested 2024-11-13T01:31:05,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465818 2024-11-13T01:31:05,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465818 2024-11-13T01:31:05,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,826 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465818) roll requested 2024-11-13T01:31:05,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465827 2024-11-13T01:31:05,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465827 2024-11-13T01:31:05,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465827) roll requested 2024-11-13T01:31:05,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465836 2024-11-13T01:31:05,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465836 2024-11-13T01:31:05,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465836) roll requested 2024-11-13T01:31:05,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465847 2024-11-13T01:31:05,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465847 2024-11-13T01:31:05,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465847) roll requested 2024-11-13T01:31:05,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465854 2024-11-13T01:31:05,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465854 2024-11-13T01:31:05,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465854) roll requested 2024-11-13T01:31:05,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465860 2024-11-13T01:31:05,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465860 2024-11-13T01:31:05,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465860) roll requested 2024-11-13T01:31:05,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465867 2024-11-13T01:31:05,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465867 2024-11-13T01:31:05,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465867) roll requested 2024-11-13T01:31:05,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465876 2024-11-13T01:31:05,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465876 2024-11-13T01:31:05,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465876) roll requested 2024-11-13T01:31:05,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465885 2024-11-13T01:31:05,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465885 2024-11-13T01:31:05,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465885) roll requested 2024-11-13T01:31:05,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465894 2024-11-13T01:31:05,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465894 2024-11-13T01:31:05,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465894) roll requested 2024-11-13T01:31:05,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465902 2024-11-13T01:31:05,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465902 2024-11-13T01:31:05,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465902) roll requested 2024-11-13T01:31:05,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465911 2024-11-13T01:31:05,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465902 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465911 2024-11-13T01:31:05,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465911) roll requested 2024-11-13T01:31:05,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465918 2024-11-13T01:31:05,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465918 2024-11-13T01:31:05,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465918) roll requested 2024-11-13T01:31:05,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465927 2024-11-13T01:31:05,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465927 2024-11-13T01:31:05,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465927) roll requested 2024-11-13T01:31:05,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465931 2024-11-13T01:31:05,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465931 2024-11-13T01:31:05,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465931) roll requested 2024-11-13T01:31:05,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465937 2024-11-13T01:31:05,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465937 2024-11-13T01:31:05,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,942 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465937) roll requested 2024-11-13T01:31:05,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465942 2024-11-13T01:31:05,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465942 2024-11-13T01:31:05,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465942) roll requested 2024-11-13T01:31:05,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465947 2024-11-13T01:31:05,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465942 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465947 2024-11-13T01:31:05,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465947) roll requested 2024-11-13T01:31:05,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465953 2024-11-13T01:31:05,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465953 2024-11-13T01:31:05,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465953) roll requested 2024-11-13T01:31:05,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465960 2024-11-13T01:31:05,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465960 2024-11-13T01:31:05,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465960) roll requested 2024-11-13T01:31:05,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465967 2024-11-13T01:31:05,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465967 2024-11-13T01:31:05,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465967) roll requested 2024-11-13T01:31:05,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465976 2024-11-13T01:31:05,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465976 2024-11-13T01:31:05,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465976) roll requested 2024-11-13T01:31:05,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465983 2024-11-13T01:31:05,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,989 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,989 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465983 2024-11-13T01:31:05,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465983) roll requested 2024-11-13T01:31:05,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465991 2024-11-13T01:31:05,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:05,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465991 2024-11-13T01:31:05,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:05,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:05,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465991) roll requested 2024-11-13T01:31:05,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461465998 2024-11-13T01:31:06,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465998 2024-11-13T01:31:06,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461465998) roll requested 2024-11-13T01:31:06,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466004 2024-11-13T01:31:06,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461465998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466004 2024-11-13T01:31:06,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466004) roll requested 2024-11-13T01:31:06,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466009 2024-11-13T01:31:06,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466009 2024-11-13T01:31:06,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466009) roll requested 2024-11-13T01:31:06,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466014 2024-11-13T01:31:06,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466014 2024-11-13T01:31:06,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466014) roll requested 2024-11-13T01:31:06,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466019 2024-11-13T01:31:06,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466019 2024-11-13T01:31:06,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,026 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466019) roll requested 2024-11-13T01:31:06,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466026 2024-11-13T01:31:06,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466026 2024-11-13T01:31:06,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,033 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466026) roll requested 2024-11-13T01:31:06,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466033 2024-11-13T01:31:06,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466033 2024-11-13T01:31:06,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466033) roll requested 2024-11-13T01:31:06,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466042 2024-11-13T01:31:06,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466033 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466042 2024-11-13T01:31:06,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466042) roll requested 2024-11-13T01:31:06,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466051 2024-11-13T01:31:06,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466051 2024-11-13T01:31:06,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466051) roll requested 2024-11-13T01:31:06,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466060 2024-11-13T01:31:06,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466060 2024-11-13T01:31:06,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466060) roll requested 2024-11-13T01:31:06,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466069 2024-11-13T01:31:06,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466060 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466069 2024-11-13T01:31:06,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,077 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466069) roll requested 2024-11-13T01:31:06,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466077 2024-11-13T01:31:06,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466077 2024-11-13T01:31:06,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466077) roll requested 2024-11-13T01:31:06,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466084 2024-11-13T01:31:06,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466084 2024-11-13T01:31:06,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,093 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466084) roll requested 2024-11-13T01:31:06,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466093 2024-11-13T01:31:06,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466093 2024-11-13T01:31:06,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466093) roll requested 2024-11-13T01:31:06,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466101 2024-11-13T01:31:06,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466101 2024-11-13T01:31:06,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466101) roll requested 2024-11-13T01:31:06,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466108 2024-11-13T01:31:06,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466101 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466108 2024-11-13T01:31:06,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466108) roll requested 2024-11-13T01:31:06,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466116 2024-11-13T01:31:06,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466116 2024-11-13T01:31:06,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466116) roll requested 2024-11-13T01:31:06,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466123 2024-11-13T01:31:06,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466123 2024-11-13T01:31:06,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466123) roll requested 2024-11-13T01:31:06,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466130 2024-11-13T01:31:06,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466130 2024-11-13T01:31:06,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466130) roll requested 2024-11-13T01:31:06,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466138 2024-11-13T01:31:06,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466138 2024-11-13T01:31:06,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,146 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466138) roll requested 2024-11-13T01:31:06,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466146 2024-11-13T01:31:06,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466146 2024-11-13T01:31:06,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466146) roll requested 2024-11-13T01:31:06,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466154 2024-11-13T01:31:06,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466154 2024-11-13T01:31:06,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,162 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466154) roll requested 2024-11-13T01:31:06,163 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466163 2024-11-13T01:31:06,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466163 2024-11-13T01:31:06,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466163) roll requested 2024-11-13T01:31:06,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466171 2024-11-13T01:31:06,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466171 2024-11-13T01:31:06,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466171) roll requested 2024-11-13T01:31:06,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466179 2024-11-13T01:31:06,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466179 2024-11-13T01:31:06,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466179) roll requested 2024-11-13T01:31:06,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466184 2024-11-13T01:31:06,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466179 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466184 2024-11-13T01:31:06,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,190 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466184) roll requested 2024-11-13T01:31:06,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466191 2024-11-13T01:31:06,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466191 2024-11-13T01:31:06,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466191) roll requested 2024-11-13T01:31:06,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466197 2024-11-13T01:31:06,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466197 2024-11-13T01:31:06,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466197) roll requested 2024-11-13T01:31:06,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466205 2024-11-13T01:31:06,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466205 2024-11-13T01:31:06,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466205) roll requested 2024-11-13T01:31:06,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466212 2024-11-13T01:31:06,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466212 2024-11-13T01:31:06,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466212) roll requested 2024-11-13T01:31:06,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466221 2024-11-13T01:31:06,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466221 2024-11-13T01:31:06,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466221) roll requested 2024-11-13T01:31:06,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466231 2024-11-13T01:31:06,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466221 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466231 2024-11-13T01:31:06,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466231) roll requested 2024-11-13T01:31:06,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466242 2024-11-13T01:31:06,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466242 2024-11-13T01:31:06,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466242) roll requested 2024-11-13T01:31:06,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466251 2024-11-13T01:31:06,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466251 2024-11-13T01:31:06,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,258 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466251) roll requested 2024-11-13T01:31:06,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466259 2024-11-13T01:31:06,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466259 2024-11-13T01:31:06,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466259) roll requested 2024-11-13T01:31:06,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466267 2024-11-13T01:31:06,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466267 2024-11-13T01:31:06,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466267) roll requested 2024-11-13T01:31:06,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466277 2024-11-13T01:31:06,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466277 2024-11-13T01:31:06,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466277) roll requested 2024-11-13T01:31:06,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466285 2024-11-13T01:31:06,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466285 2024-11-13T01:31:06,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466285) roll requested 2024-11-13T01:31:06,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466293 2024-11-13T01:31:06,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466293 2024-11-13T01:31:06,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466293) roll requested 2024-11-13T01:31:06,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466301 2024-11-13T01:31:06,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466301 2024-11-13T01:31:06,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466301) roll requested 2024-11-13T01:31:06,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466310 2024-11-13T01:31:06,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466310 2024-11-13T01:31:06,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,316 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466310) roll requested 2024-11-13T01:31:06,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466316 2024-11-13T01:31:06,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466316 2024-11-13T01:31:06,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,321 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466316) roll requested 2024-11-13T01:31:06,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466321 2024-11-13T01:31:06,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466321 2024-11-13T01:31:06,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466321) roll requested 2024-11-13T01:31:06,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466327 2024-11-13T01:31:06,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466327 2024-11-13T01:31:06,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466327) roll requested 2024-11-13T01:31:06,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466334 2024-11-13T01:31:06,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466334 2024-11-13T01:31:06,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466334) roll requested 2024-11-13T01:31:06,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466343 2024-11-13T01:31:06,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466343 2024-11-13T01:31:06,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466343) roll requested 2024-11-13T01:31:06,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466351 2024-11-13T01:31:06,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466343 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466351 2024-11-13T01:31:06,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466351) roll requested 2024-11-13T01:31:06,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466360 2024-11-13T01:31:06,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,368 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,368 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,368 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466360 2024-11-13T01:31:06,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466360) roll requested 2024-11-13T01:31:06,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466370 2024-11-13T01:31:06,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466360 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466370 2024-11-13T01:31:06,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466370) roll requested 2024-11-13T01:31:06,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466380 2024-11-13T01:31:06,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466380 2024-11-13T01:31:06,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466380) roll requested 2024-11-13T01:31:06,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466389 2024-11-13T01:31:06,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,397 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466389 2024-11-13T01:31:06,398 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,398 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466389) roll requested 2024-11-13T01:31:06,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466398 2024-11-13T01:31:06,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466398 2024-11-13T01:31:06,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466398) roll requested 2024-11-13T01:31:06,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466406 2024-11-13T01:31:06,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466398 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466406 2024-11-13T01:31:06,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466406) roll requested 2024-11-13T01:31:06,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466414 2024-11-13T01:31:06,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466414 2024-11-13T01:31:06,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466414) roll requested 2024-11-13T01:31:06,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466423 2024-11-13T01:31:06,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466423 2024-11-13T01:31:06,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466423) roll requested 2024-11-13T01:31:06,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466432 2024-11-13T01:31:06,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,440 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466423 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466432 2024-11-13T01:31:06,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,441 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466432) roll requested 2024-11-13T01:31:06,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466441 2024-11-13T01:31:06,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466441 2024-11-13T01:31:06,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466441) roll requested 2024-11-13T01:31:06,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466451 2024-11-13T01:31:06,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466441 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466451 2024-11-13T01:31:06,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466451) roll requested 2024-11-13T01:31:06,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466459 2024-11-13T01:31:06,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466451 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466459 2024-11-13T01:31:06,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,467 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,467 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466459) roll requested 2024-11-13T01:31:06,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466467 2024-11-13T01:31:06,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466467 2024-11-13T01:31:06,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466467) roll requested 2024-11-13T01:31:06,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466476 2024-11-13T01:31:06,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466467 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466476 2024-11-13T01:31:06,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466476) roll requested 2024-11-13T01:31:06,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466484 2024-11-13T01:31:06,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466484 2024-11-13T01:31:06,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466484) roll requested 2024-11-13T01:31:06,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466490 2024-11-13T01:31:06,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466490 2024-11-13T01:31:06,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466490) roll requested 2024-11-13T01:31:06,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466496 2024-11-13T01:31:06,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466496 2024-11-13T01:31:06,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466496) roll requested 2024-11-13T01:31:06,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466502 2024-11-13T01:31:06,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466502 2024-11-13T01:31:06,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466502) roll requested 2024-11-13T01:31:06,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466515 2024-11-13T01:31:06,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466515 2024-11-13T01:31:06,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466515) roll requested 2024-11-13T01:31:06,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466525 2024-11-13T01:31:06,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466525 2024-11-13T01:31:06,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466525) roll requested 2024-11-13T01:31:06,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466535 2024-11-13T01:31:06,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466535 2024-11-13T01:31:06,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466535) roll requested 2024-11-13T01:31:06,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466542 2024-11-13T01:31:06,550 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,550 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466542 2024-11-13T01:31:06,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,551 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466542) roll requested 2024-11-13T01:31:06,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466551 2024-11-13T01:31:06,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466551 2024-11-13T01:31:06,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466551) roll requested 2024-11-13T01:31:06,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466556 2024-11-13T01:31:06,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466556 2024-11-13T01:31:06,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466556) roll requested 2024-11-13T01:31:06,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466564 2024-11-13T01:31:06,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466564 2024-11-13T01:31:06,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466564) roll requested 2024-11-13T01:31:06,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466572 2024-11-13T01:31:06,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466572 2024-11-13T01:31:06,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466572) roll requested 2024-11-13T01:31:06,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466580 2024-11-13T01:31:06,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466580 2024-11-13T01:31:06,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466580) roll requested 2024-11-13T01:31:06,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466588 2024-11-13T01:31:06,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466588 2024-11-13T01:31:06,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466588) roll requested 2024-11-13T01:31:06,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466594 2024-11-13T01:31:06,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466594 2024-11-13T01:31:06,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466594) roll requested 2024-11-13T01:31:06,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466599 2024-11-13T01:31:06,603 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,603 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466599 2024-11-13T01:31:06,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466599) roll requested 2024-11-13T01:31:06,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466604 2024-11-13T01:31:06,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466604 2024-11-13T01:31:06,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466604) roll requested 2024-11-13T01:31:06,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466609 2024-11-13T01:31:06,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466609 2024-11-13T01:31:06,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466609) roll requested 2024-11-13T01:31:06,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466617 2024-11-13T01:31:06,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466617 2024-11-13T01:31:06,622 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,622 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,622 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466617) roll requested 2024-11-13T01:31:06,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466622 2024-11-13T01:31:06,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466622 2024-11-13T01:31:06,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466622) roll requested 2024-11-13T01:31:06,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466627 2024-11-13T01:31:06,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466627 2024-11-13T01:31:06,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466627) roll requested 2024-11-13T01:31:06,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466633 2024-11-13T01:31:06,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466633 2024-11-13T01:31:06,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466633) roll requested 2024-11-13T01:31:06,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466638 2024-11-13T01:31:06,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466638 2024-11-13T01:31:06,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466638) roll requested 2024-11-13T01:31:06,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466643 2024-11-13T01:31:06,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466643 2024-11-13T01:31:06,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,648 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466643) roll requested 2024-11-13T01:31:06,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466648 2024-11-13T01:31:06,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466648 2024-11-13T01:31:06,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466648) roll requested 2024-11-13T01:31:06,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466654 2024-11-13T01:31:06,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466654 2024-11-13T01:31:06,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,661 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466654) roll requested 2024-11-13T01:31:06,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466662 2024-11-13T01:31:06,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466662 2024-11-13T01:31:06,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466662) roll requested 2024-11-13T01:31:06,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466667 2024-11-13T01:31:06,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466667 2024-11-13T01:31:06,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466667) roll requested 2024-11-13T01:31:06,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466672 2024-11-13T01:31:06,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466672 2024-11-13T01:31:06,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466672) roll requested 2024-11-13T01:31:06,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466677 2024-11-13T01:31:06,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466677 2024-11-13T01:31:06,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466677) roll requested 2024-11-13T01:31:06,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466683 2024-11-13T01:31:06,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466683 2024-11-13T01:31:06,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466683) roll requested 2024-11-13T01:31:06,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466689 2024-11-13T01:31:06,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466689 2024-11-13T01:31:06,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466689) roll requested 2024-11-13T01:31:06,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466694 2024-11-13T01:31:06,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466694 2024-11-13T01:31:06,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466694) roll requested 2024-11-13T01:31:06,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466699 2024-11-13T01:31:06,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466694 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466699 2024-11-13T01:31:06,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466699) roll requested 2024-11-13T01:31:06,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466705 2024-11-13T01:31:06,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466705 2024-11-13T01:31:06,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466705) roll requested 2024-11-13T01:31:06,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466712 2024-11-13T01:31:06,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466712 2024-11-13T01:31:06,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466712) roll requested 2024-11-13T01:31:06,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466720 2024-11-13T01:31:06,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466720 2024-11-13T01:31:06,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,728 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466720) roll requested 2024-11-13T01:31:06,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466728 2024-11-13T01:31:06,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466720 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466728 2024-11-13T01:31:06,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466728) roll requested 2024-11-13T01:31:06,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466736 2024-11-13T01:31:06,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466728 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466736 2024-11-13T01:31:06,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466736) roll requested 2024-11-13T01:31:06,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466745 2024-11-13T01:31:06,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466736 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466745 2024-11-13T01:31:06,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,751 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466745) roll requested 2024-11-13T01:31:06,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466751 2024-11-13T01:31:06,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466751 2024-11-13T01:31:06,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466751) roll requested 2024-11-13T01:31:06,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466758 2024-11-13T01:31:06,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466751 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466758 2024-11-13T01:31:06,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466758) roll requested 2024-11-13T01:31:06,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466764 2024-11-13T01:31:06,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466764 2024-11-13T01:31:06,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,771 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466764) roll requested 2024-11-13T01:31:06,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466771 2024-11-13T01:31:06,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466771 2024-11-13T01:31:06,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466771) roll requested 2024-11-13T01:31:06,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466779 2024-11-13T01:31:06,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466779 2024-11-13T01:31:06,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466779) roll requested 2024-11-13T01:31:06,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466787 2024-11-13T01:31:06,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466787 2024-11-13T01:31:06,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466787) roll requested 2024-11-13T01:31:06,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466795 2024-11-13T01:31:06,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466787 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466795 2024-11-13T01:31:06,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466795) roll requested 2024-11-13T01:31:06,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466803 2024-11-13T01:31:06,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466803 2024-11-13T01:31:06,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466803) roll requested 2024-11-13T01:31:06,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466810 2024-11-13T01:31:06,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466810 2024-11-13T01:31:06,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,819 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466810) roll requested 2024-11-13T01:31:06,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466819 2024-11-13T01:31:06,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466810 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466819 2024-11-13T01:31:06,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466819) roll requested 2024-11-13T01:31:06,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466829 2024-11-13T01:31:06,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466829 2024-11-13T01:31:06,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466829) roll requested 2024-11-13T01:31:06,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466836 2024-11-13T01:31:06,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466836 2024-11-13T01:31:06,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466836) roll requested 2024-11-13T01:31:06,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466842 2024-11-13T01:31:06,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466842 2024-11-13T01:31:06,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466842) roll requested 2024-11-13T01:31:06,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466850 2024-11-13T01:31:06,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466850 2024-11-13T01:31:06,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466850) roll requested 2024-11-13T01:31:06,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466858 2024-11-13T01:31:06,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466858 2024-11-13T01:31:06,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,866 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466858) roll requested 2024-11-13T01:31:06,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466866 2024-11-13T01:31:06,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466866 2024-11-13T01:31:06,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466866) roll requested 2024-11-13T01:31:06,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466875 2024-11-13T01:31:06,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466866 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466875 2024-11-13T01:31:06,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466875) roll requested 2024-11-13T01:31:06,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466882 2024-11-13T01:31:06,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466882 2024-11-13T01:31:06,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466882) roll requested 2024-11-13T01:31:06,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466890 2024-11-13T01:31:06,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466890 2024-11-13T01:31:06,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466890) roll requested 2024-11-13T01:31:06,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466901 2024-11-13T01:31:06,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466901 2024-11-13T01:31:06,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466901) roll requested 2024-11-13T01:31:06,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466906 2024-11-13T01:31:06,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466906 2024-11-13T01:31:06,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466906) roll requested 2024-11-13T01:31:06,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466910 2024-11-13T01:31:06,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466910 2024-11-13T01:31:06,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466910) roll requested 2024-11-13T01:31:06,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466916 2024-11-13T01:31:06,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466916 2024-11-13T01:31:06,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466916) roll requested 2024-11-13T01:31:06,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466923 2024-11-13T01:31:06,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466923 2024-11-13T01:31:06,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466923) roll requested 2024-11-13T01:31:06,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466932 2024-11-13T01:31:06,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466932 2024-11-13T01:31:06,941 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,941 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466932) roll requested 2024-11-13T01:31:06,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466941 2024-11-13T01:31:06,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466941 2024-11-13T01:31:06,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466941) roll requested 2024-11-13T01:31:06,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466947 2024-11-13T01:31:06,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466947 2024-11-13T01:31:06,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466947) roll requested 2024-11-13T01:31:06,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466954 2024-11-13T01:31:06,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466954 2024-11-13T01:31:06,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466954) roll requested 2024-11-13T01:31:06,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466960 2024-11-13T01:31:06,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466960 2024-11-13T01:31:06,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466960) roll requested 2024-11-13T01:31:06,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466969 2024-11-13T01:31:06,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466969 2024-11-13T01:31:06,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466969) roll requested 2024-11-13T01:31:06,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466978 2024-11-13T01:31:06,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466978 2024-11-13T01:31:06,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466978) roll requested 2024-11-13T01:31:06,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466988 2024-11-13T01:31:06,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:06,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466988 2024-11-13T01:31:06,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:06,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:06,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466988) roll requested 2024-11-13T01:31:06,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461466997 2024-11-13T01:31:07,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466997 2024-11-13T01:31:07,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461466997) roll requested 2024-11-13T01:31:07,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467004 2024-11-13T01:31:07,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461466997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467004 2024-11-13T01:31:07,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467004) roll requested 2024-11-13T01:31:07,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467011 2024-11-13T01:31:07,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467011 2024-11-13T01:31:07,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,018 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467011) roll requested 2024-11-13T01:31:07,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467018 2024-11-13T01:31:07,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467011 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467018 2024-11-13T01:31:07,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467018) roll requested 2024-11-13T01:31:07,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467023 2024-11-13T01:31:07,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,028 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467023 2024-11-13T01:31:07,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,029 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467023) roll requested 2024-11-13T01:31:07,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467029 2024-11-13T01:31:07,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467029 2024-11-13T01:31:07,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467029) roll requested 2024-11-13T01:31:07,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467035 2024-11-13T01:31:07,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467035 2024-11-13T01:31:07,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,044 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467035) roll requested 2024-11-13T01:31:07,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467044 2024-11-13T01:31:07,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467044 2024-11-13T01:31:07,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467044) roll requested 2024-11-13T01:31:07,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467053 2024-11-13T01:31:07,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467053 2024-11-13T01:31:07,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467053) roll requested 2024-11-13T01:31:07,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467063 2024-11-13T01:31:07,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467063 2024-11-13T01:31:07,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467063) roll requested 2024-11-13T01:31:07,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467068 2024-11-13T01:31:07,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467068 2024-11-13T01:31:07,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467068) roll requested 2024-11-13T01:31:07,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467076 2024-11-13T01:31:07,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467076 2024-11-13T01:31:07,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467076) roll requested 2024-11-13T01:31:07,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467083 2024-11-13T01:31:07,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467083 2024-11-13T01:31:07,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467083) roll requested 2024-11-13T01:31:07,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467088 2024-11-13T01:31:07,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467088 2024-11-13T01:31:07,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467088) roll requested 2024-11-13T01:31:07,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467093 2024-11-13T01:31:07,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467093 2024-11-13T01:31:07,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467093) roll requested 2024-11-13T01:31:07,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467098 2024-11-13T01:31:07,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467098 2024-11-13T01:31:07,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467098) roll requested 2024-11-13T01:31:07,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467105 2024-11-13T01:31:07,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467105 2024-11-13T01:31:07,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467105) roll requested 2024-11-13T01:31:07,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467113 2024-11-13T01:31:07,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467113 2024-11-13T01:31:07,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467113) roll requested 2024-11-13T01:31:07,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467119 2024-11-13T01:31:07,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467119 2024-11-13T01:31:07,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467119) roll requested 2024-11-13T01:31:07,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467124 2024-11-13T01:31:07,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467124 2024-11-13T01:31:07,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467124) roll requested 2024-11-13T01:31:07,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467129 2024-11-13T01:31:07,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467129 2024-11-13T01:31:07,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467129) roll requested 2024-11-13T01:31:07,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467135 2024-11-13T01:31:07,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467135 2024-11-13T01:31:07,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467135) roll requested 2024-11-13T01:31:07,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467143 2024-11-13T01:31:07,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467143 2024-11-13T01:31:07,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467143) roll requested 2024-11-13T01:31:07,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467151 2024-11-13T01:31:07,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467151 2024-11-13T01:31:07,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467151) roll requested 2024-11-13T01:31:07,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467158 2024-11-13T01:31:07,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467158 2024-11-13T01:31:07,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467158) roll requested 2024-11-13T01:31:07,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467166 2024-11-13T01:31:07,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467166 2024-11-13T01:31:07,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467166) roll requested 2024-11-13T01:31:07,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467175 2024-11-13T01:31:07,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467175 2024-11-13T01:31:07,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467175) roll requested 2024-11-13T01:31:07,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467182 2024-11-13T01:31:07,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467182 2024-11-13T01:31:07,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467182) roll requested 2024-11-13T01:31:07,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467189 2024-11-13T01:31:07,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467189 2024-11-13T01:31:07,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467189) roll requested 2024-11-13T01:31:07,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467199 2024-11-13T01:31:07,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467199 2024-11-13T01:31:07,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467199) roll requested 2024-11-13T01:31:07,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467208 2024-11-13T01:31:07,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467199 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467208 2024-11-13T01:31:07,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467208) roll requested 2024-11-13T01:31:07,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467219 2024-11-13T01:31:07,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467219 2024-11-13T01:31:07,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467219) roll requested 2024-11-13T01:31:07,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467227 2024-11-13T01:31:07,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467227 2024-11-13T01:31:07,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467227) roll requested 2024-11-13T01:31:07,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467236 2024-11-13T01:31:07,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467227 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467236 2024-11-13T01:31:07,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467236) roll requested 2024-11-13T01:31:07,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467247 2024-11-13T01:31:07,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467247 2024-11-13T01:31:07,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467247) roll requested 2024-11-13T01:31:07,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467256 2024-11-13T01:31:07,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467247 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467256 2024-11-13T01:31:07,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467256) roll requested 2024-11-13T01:31:07,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467263 2024-11-13T01:31:07,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467263 2024-11-13T01:31:07,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,274 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467263) roll requested 2024-11-13T01:31:07,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467274 2024-11-13T01:31:07,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467274 2024-11-13T01:31:07,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467274) roll requested 2024-11-13T01:31:07,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467279 2024-11-13T01:31:07,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467274 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467279 2024-11-13T01:31:07,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467279) roll requested 2024-11-13T01:31:07,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467284 2024-11-13T01:31:07,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467284 2024-11-13T01:31:07,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467284) roll requested 2024-11-13T01:31:07,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467292 2024-11-13T01:31:07,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467292 2024-11-13T01:31:07,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467292) roll requested 2024-11-13T01:31:07,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467299 2024-11-13T01:31:07,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467299 2024-11-13T01:31:07,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,307 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467299) roll requested 2024-11-13T01:31:07,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467307 2024-11-13T01:31:07,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,314 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467307 2024-11-13T01:31:07,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467307) roll requested 2024-11-13T01:31:07,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467316 2024-11-13T01:31:07,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467316 2024-11-13T01:31:07,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467316) roll requested 2024-11-13T01:31:07,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467325 2024-11-13T01:31:07,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,331 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,331 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467325 2024-11-13T01:31:07,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467325) roll requested 2024-11-13T01:31:07,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467331 2024-11-13T01:31:07,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467331 2024-11-13T01:31:07,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,339 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467331) roll requested 2024-11-13T01:31:07,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467339 2024-11-13T01:31:07,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467339 2024-11-13T01:31:07,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,346 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467339) roll requested 2024-11-13T01:31:07,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467346 2024-11-13T01:31:07,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467339 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467346 2024-11-13T01:31:07,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467346) roll requested 2024-11-13T01:31:07,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467355 2024-11-13T01:31:07,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467346 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467355 2024-11-13T01:31:07,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467355) roll requested 2024-11-13T01:31:07,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467362 2024-11-13T01:31:07,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467362 2024-11-13T01:31:07,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467362) roll requested 2024-11-13T01:31:07,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467368 2024-11-13T01:31:07,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467368 2024-11-13T01:31:07,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467368) roll requested 2024-11-13T01:31:07,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467375 2024-11-13T01:31:07,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467375 2024-11-13T01:31:07,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467375) roll requested 2024-11-13T01:31:07,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467384 2024-11-13T01:31:07,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467384 2024-11-13T01:31:07,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467384) roll requested 2024-11-13T01:31:07,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467394 2024-11-13T01:31:07,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467394 2024-11-13T01:31:07,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,402 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467394) roll requested 2024-11-13T01:31:07,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467402 2024-11-13T01:31:07,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467402 2024-11-13T01:31:07,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467402) roll requested 2024-11-13T01:31:07,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467409 2024-11-13T01:31:07,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467402 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467409 2024-11-13T01:31:07,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467409) roll requested 2024-11-13T01:31:07,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467416 2024-11-13T01:31:07,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467416 2024-11-13T01:31:07,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467416) roll requested 2024-11-13T01:31:07,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467424 2024-11-13T01:31:07,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467424 2024-11-13T01:31:07,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467424) roll requested 2024-11-13T01:31:07,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467432 2024-11-13T01:31:07,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467432 2024-11-13T01:31:07,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467432) roll requested 2024-11-13T01:31:07,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467439 2024-11-13T01:31:07,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467439 2024-11-13T01:31:07,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,447 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467439) roll requested 2024-11-13T01:31:07,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467448 2024-11-13T01:31:07,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,455 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,455 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,455 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467448 2024-11-13T01:31:07,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467448) roll requested 2024-11-13T01:31:07,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467457 2024-11-13T01:31:07,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467457 2024-11-13T01:31:07,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467457) roll requested 2024-11-13T01:31:07,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467464 2024-11-13T01:31:07,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467464 2024-11-13T01:31:07,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467464) roll requested 2024-11-13T01:31:07,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467472 2024-11-13T01:31:07,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467472 2024-11-13T01:31:07,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467472) roll requested 2024-11-13T01:31:07,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467480 2024-11-13T01:31:07,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467480 2024-11-13T01:31:07,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,489 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,489 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467480) roll requested 2024-11-13T01:31:07,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467489 2024-11-13T01:31:07,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467489 2024-11-13T01:31:07,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467489) roll requested 2024-11-13T01:31:07,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467498 2024-11-13T01:31:07,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467498 2024-11-13T01:31:07,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,506 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467498) roll requested 2024-11-13T01:31:07,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467506 2024-11-13T01:31:07,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467498 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467506 2024-11-13T01:31:07,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467506) roll requested 2024-11-13T01:31:07,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467516 2024-11-13T01:31:07,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467516 2024-11-13T01:31:07,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467516) roll requested 2024-11-13T01:31:07,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467524 2024-11-13T01:31:07,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467524 2024-11-13T01:31:07,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467524) roll requested 2024-11-13T01:31:07,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467533 2024-11-13T01:31:07,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467533 2024-11-13T01:31:07,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467533) roll requested 2024-11-13T01:31:07,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467541 2024-11-13T01:31:07,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467541 2024-11-13T01:31:07,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467541) roll requested 2024-11-13T01:31:07,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467550 2024-11-13T01:31:07,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467550 2024-11-13T01:31:07,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467550) roll requested 2024-11-13T01:31:07,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467559 2024-11-13T01:31:07,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467550 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467559 2024-11-13T01:31:07,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467559) roll requested 2024-11-13T01:31:07,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467568 2024-11-13T01:31:07,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467559 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467568 2024-11-13T01:31:07,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467568) roll requested 2024-11-13T01:31:07,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467576 2024-11-13T01:31:07,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467576 2024-11-13T01:31:07,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467576) roll requested 2024-11-13T01:31:07,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467584 2024-11-13T01:31:07,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467584 2024-11-13T01:31:07,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467584) roll requested 2024-11-13T01:31:07,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467590 2024-11-13T01:31:07,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467590 2024-11-13T01:31:07,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467590) roll requested 2024-11-13T01:31:07,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467597 2024-11-13T01:31:07,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467590 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467597 2024-11-13T01:31:07,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467597) roll requested 2024-11-13T01:31:07,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467605 2024-11-13T01:31:07,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467605 2024-11-13T01:31:07,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,612 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467605) roll requested 2024-11-13T01:31:07,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467612 2024-11-13T01:31:07,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467612 2024-11-13T01:31:07,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467612) roll requested 2024-11-13T01:31:07,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467620 2024-11-13T01:31:07,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467612 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467620 2024-11-13T01:31:07,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467620) roll requested 2024-11-13T01:31:07,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467629 2024-11-13T01:31:07,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467629 2024-11-13T01:31:07,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,635 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467629) roll requested 2024-11-13T01:31:07,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467635 2024-11-13T01:31:07,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467635 2024-11-13T01:31:07,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,641 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467635) roll requested 2024-11-13T01:31:07,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467641 2024-11-13T01:31:07,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467635 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467641 2024-11-13T01:31:07,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467641) roll requested 2024-11-13T01:31:07,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467650 2024-11-13T01:31:07,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467641 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467650 2024-11-13T01:31:07,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467650) roll requested 2024-11-13T01:31:07,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467659 2024-11-13T01:31:07,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467659 2024-11-13T01:31:07,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467659) roll requested 2024-11-13T01:31:07,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467668 2024-11-13T01:31:07,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467668 2024-11-13T01:31:07,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,680 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467668) roll requested 2024-11-13T01:31:07,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467680 2024-11-13T01:31:07,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467680 2024-11-13T01:31:07,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467680) roll requested 2024-11-13T01:31:07,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467685 2024-11-13T01:31:07,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467680 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467685 2024-11-13T01:31:07,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467685) roll requested 2024-11-13T01:31:07,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467692 2024-11-13T01:31:07,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467685 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467692 2024-11-13T01:31:07,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467692) roll requested 2024-11-13T01:31:07,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467699 2024-11-13T01:31:07,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467699 2024-11-13T01:31:07,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467699) roll requested 2024-11-13T01:31:07,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467707 2024-11-13T01:31:07,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467707 2024-11-13T01:31:07,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,715 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467707) roll requested 2024-11-13T01:31:07,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467715 2024-11-13T01:31:07,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467715 2024-11-13T01:31:07,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467715) roll requested 2024-11-13T01:31:07,723 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467723 2024-11-13T01:31:07,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467715 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467723 2024-11-13T01:31:07,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467723) roll requested 2024-11-13T01:31:07,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467731 2024-11-13T01:31:07,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467723 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467731 2024-11-13T01:31:07,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467731) roll requested 2024-11-13T01:31:07,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467740 2024-11-13T01:31:07,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467740 2024-11-13T01:31:07,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467740) roll requested 2024-11-13T01:31:07,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467747 2024-11-13T01:31:07,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467747 2024-11-13T01:31:07,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467747) roll requested 2024-11-13T01:31:07,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467753 2024-11-13T01:31:07,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467753 2024-11-13T01:31:07,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467753) roll requested 2024-11-13T01:31:07,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467760 2024-11-13T01:31:07,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467760 2024-11-13T01:31:07,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467760) roll requested 2024-11-13T01:31:07,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467767 2024-11-13T01:31:07,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467767 2024-11-13T01:31:07,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467767) roll requested 2024-11-13T01:31:07,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467773 2024-11-13T01:31:07,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467773 2024-11-13T01:31:07,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467773) roll requested 2024-11-13T01:31:07,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467779 2024-11-13T01:31:07,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467779 2024-11-13T01:31:07,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467779) roll requested 2024-11-13T01:31:07,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467786 2024-11-13T01:31:07,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467786 2024-11-13T01:31:07,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467786) roll requested 2024-11-13T01:31:07,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467793 2024-11-13T01:31:07,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467793 2024-11-13T01:31:07,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467793) roll requested 2024-11-13T01:31:07,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467800 2024-11-13T01:31:07,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467800 2024-11-13T01:31:07,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,807 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467800) roll requested 2024-11-13T01:31:07,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467808 2024-11-13T01:31:07,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467808 2024-11-13T01:31:07,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467808) roll requested 2024-11-13T01:31:07,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467814 2024-11-13T01:31:07,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467808 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467814 2024-11-13T01:31:07,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467814) roll requested 2024-11-13T01:31:07,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467824 2024-11-13T01:31:07,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467824 2024-11-13T01:31:07,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467824) roll requested 2024-11-13T01:31:07,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467833 2024-11-13T01:31:07,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467833 2024-11-13T01:31:07,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,841 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467833) roll requested 2024-11-13T01:31:07,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467842 2024-11-13T01:31:07,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467833 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467842 2024-11-13T01:31:07,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467842) roll requested 2024-11-13T01:31:07,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467851 2024-11-13T01:31:07,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467851 2024-11-13T01:31:07,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467851) roll requested 2024-11-13T01:31:07,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467859 2024-11-13T01:31:07,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467851 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467859 2024-11-13T01:31:07,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467859) roll requested 2024-11-13T01:31:07,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467868 2024-11-13T01:31:07,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,875 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467868 2024-11-13T01:31:07,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467868) roll requested 2024-11-13T01:31:07,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467876 2024-11-13T01:31:07,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467876 2024-11-13T01:31:07,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467876) roll requested 2024-11-13T01:31:07,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467884 2024-11-13T01:31:07,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467884 2024-11-13T01:31:07,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467884) roll requested 2024-11-13T01:31:07,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467892 2024-11-13T01:31:07,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467892 2024-11-13T01:31:07,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467892) roll requested 2024-11-13T01:31:07,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467899 2024-11-13T01:31:07,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467899 2024-11-13T01:31:07,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,905 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467899) roll requested 2024-11-13T01:31:07,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467906 2024-11-13T01:31:07,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467906 2024-11-13T01:31:07,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467906) roll requested 2024-11-13T01:31:07,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467914 2024-11-13T01:31:07,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467906 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467914 2024-11-13T01:31:07,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467914) roll requested 2024-11-13T01:31:07,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467922 2024-11-13T01:31:07,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467922 2024-11-13T01:31:07,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467922) roll requested 2024-11-13T01:31:07,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467930 2024-11-13T01:31:07,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467930 2024-11-13T01:31:07,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467930) roll requested 2024-11-13T01:31:07,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467938 2024-11-13T01:31:07,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467938 2024-11-13T01:31:07,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467938) roll requested 2024-11-13T01:31:07,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467947 2024-11-13T01:31:07,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467947 2024-11-13T01:31:07,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,954 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467947) roll requested 2024-11-13T01:31:07,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467954 2024-11-13T01:31:07,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467954 2024-11-13T01:31:07,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467954) roll requested 2024-11-13T01:31:07,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467962 2024-11-13T01:31:07,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467962 2024-11-13T01:31:07,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467962) roll requested 2024-11-13T01:31:07,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467972 2024-11-13T01:31:07,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467972 2024-11-13T01:31:07,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467972) roll requested 2024-11-13T01:31:07,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467979 2024-11-13T01:31:07,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467979 2024-11-13T01:31:07,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,984 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467979) roll requested 2024-11-13T01:31:07,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467984 2024-11-13T01:31:07,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467984 2024-11-13T01:31:07,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:07,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:07,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467984) roll requested 2024-11-13T01:31:07,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461467993 2024-11-13T01:31:07,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:07,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467984 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467993 2024-11-13T01:31:08,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461467993) roll requested 2024-11-13T01:31:08,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468000 2024-11-13T01:31:08,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,005 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461467993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468000 2024-11-13T01:31:08,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468000) roll requested 2024-11-13T01:31:08,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468006 2024-11-13T01:31:08,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468006 2024-11-13T01:31:08,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468006) roll requested 2024-11-13T01:31:08,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468014 2024-11-13T01:31:08,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468014 2024-11-13T01:31:08,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,023 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468014) roll requested 2024-11-13T01:31:08,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468023 2024-11-13T01:31:08,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468023 2024-11-13T01:31:08,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468023) roll requested 2024-11-13T01:31:08,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468032 2024-11-13T01:31:08,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468032 2024-11-13T01:31:08,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468032) roll requested 2024-11-13T01:31:08,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468040 2024-11-13T01:31:08,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468040 2024-11-13T01:31:08,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468040) roll requested 2024-11-13T01:31:08,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468048 2024-11-13T01:31:08,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468040 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468048 2024-11-13T01:31:08,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468048) roll requested 2024-11-13T01:31:08,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468056 2024-11-13T01:31:08,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468056 2024-11-13T01:31:08,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468056) roll requested 2024-11-13T01:31:08,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468063 2024-11-13T01:31:08,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,070 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,070 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,070 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468063 2024-11-13T01:31:08,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468063) roll requested 2024-11-13T01:31:08,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468071 2024-11-13T01:31:08,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468071 2024-11-13T01:31:08,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468071) roll requested 2024-11-13T01:31:08,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468080 2024-11-13T01:31:08,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468080 2024-11-13T01:31:08,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468080) roll requested 2024-11-13T01:31:08,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468088 2024-11-13T01:31:08,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468088 2024-11-13T01:31:08,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468088) roll requested 2024-11-13T01:31:08,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468094 2024-11-13T01:31:08,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468094 2024-11-13T01:31:08,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,100 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468094) roll requested 2024-11-13T01:31:08,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468100 2024-11-13T01:31:08,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468100 2024-11-13T01:31:08,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468100) roll requested 2024-11-13T01:31:08,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468110 2024-11-13T01:31:08,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468100 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468110 2024-11-13T01:31:08,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468110) roll requested 2024-11-13T01:31:08,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468118 2024-11-13T01:31:08,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468118 2024-11-13T01:31:08,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468118) roll requested 2024-11-13T01:31:08,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468126 2024-11-13T01:31:08,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468126 2024-11-13T01:31:08,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468126) roll requested 2024-11-13T01:31:08,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468133 2024-11-13T01:31:08,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468133 2024-11-13T01:31:08,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,142 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,142 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468133) roll requested 2024-11-13T01:31:08,142 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468142 2024-11-13T01:31:08,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468142 2024-11-13T01:31:08,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468142) roll requested 2024-11-13T01:31:08,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468151 2024-11-13T01:31:08,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468142 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468151 2024-11-13T01:31:08,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468151) roll requested 2024-11-13T01:31:08,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468158 2024-11-13T01:31:08,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468151 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468158 2024-11-13T01:31:08,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468158) roll requested 2024-11-13T01:31:08,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468168 2024-11-13T01:31:08,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468168 2024-11-13T01:31:08,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468168) roll requested 2024-11-13T01:31:08,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468175 2024-11-13T01:31:08,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468175 2024-11-13T01:31:08,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,183 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468175) roll requested 2024-11-13T01:31:08,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468184 2024-11-13T01:31:08,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468184 2024-11-13T01:31:08,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468184) roll requested 2024-11-13T01:31:08,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468192 2024-11-13T01:31:08,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468192 2024-11-13T01:31:08,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468192) roll requested 2024-11-13T01:31:08,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468197 2024-11-13T01:31:08,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468197 2024-11-13T01:31:08,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468197) roll requested 2024-11-13T01:31:08,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468202 2024-11-13T01:31:08,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468202 2024-11-13T01:31:08,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,206 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468202) roll requested 2024-11-13T01:31:08,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468206 2024-11-13T01:31:08,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468206 2024-11-13T01:31:08,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,212 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468206) roll requested 2024-11-13T01:31:08,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468212 2024-11-13T01:31:08,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468206 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468212 2024-11-13T01:31:08,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,219 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468212) roll requested 2024-11-13T01:31:08,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468219 2024-11-13T01:31:08,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468219 2024-11-13T01:31:08,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468219) roll requested 2024-11-13T01:31:08,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468226 2024-11-13T01:31:08,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468219 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468226 2024-11-13T01:31:08,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,235 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468226) roll requested 2024-11-13T01:31:08,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468235 2024-11-13T01:31:08,240 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468235 2024-11-13T01:31:08,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468235) roll requested 2024-11-13T01:31:08,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468241 2024-11-13T01:31:08,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468235 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468241 2024-11-13T01:31:08,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,247 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468241) roll requested 2024-11-13T01:31:08,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468248 2024-11-13T01:31:08,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468248 2024-11-13T01:31:08,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,255 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468248) roll requested 2024-11-13T01:31:08,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468255 2024-11-13T01:31:08,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468255 2024-11-13T01:31:08,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468255) roll requested 2024-11-13T01:31:08,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468263 2024-11-13T01:31:08,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468255 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468263 2024-11-13T01:31:08,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468263) roll requested 2024-11-13T01:31:08,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468272 2024-11-13T01:31:08,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468272 2024-11-13T01:31:08,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468272) roll requested 2024-11-13T01:31:08,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468280 2024-11-13T01:31:08,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468280 2024-11-13T01:31:08,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468280) roll requested 2024-11-13T01:31:08,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468287 2024-11-13T01:31:08,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,293 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,293 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,293 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468287 2024-11-13T01:31:08,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,294 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468287) roll requested 2024-11-13T01:31:08,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468294 2024-11-13T01:31:08,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468294 2024-11-13T01:31:08,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468294) roll requested 2024-11-13T01:31:08,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468302 2024-11-13T01:31:08,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468294 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468302 2024-11-13T01:31:08,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468302) roll requested 2024-11-13T01:31:08,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468310 2024-11-13T01:31:08,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468302 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468310 2024-11-13T01:31:08,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468310) roll requested 2024-11-13T01:31:08,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468320 2024-11-13T01:31:08,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468320 2024-11-13T01:31:08,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468320) roll requested 2024-11-13T01:31:08,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468330 2024-11-13T01:31:08,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468330 2024-11-13T01:31:08,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,338 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468330) roll requested 2024-11-13T01:31:08,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468338 2024-11-13T01:31:08,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468338 2024-11-13T01:31:08,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468338) roll requested 2024-11-13T01:31:08,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468348 2024-11-13T01:31:08,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468348 2024-11-13T01:31:08,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468348) roll requested 2024-11-13T01:31:08,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468358 2024-11-13T01:31:08,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468358 2024-11-13T01:31:08,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468358) roll requested 2024-11-13T01:31:08,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468368 2024-11-13T01:31:08,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468368 2024-11-13T01:31:08,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468368) roll requested 2024-11-13T01:31:08,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468376 2024-11-13T01:31:08,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468376 2024-11-13T01:31:08,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,382 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468376) roll requested 2024-11-13T01:31:08,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468382 2024-11-13T01:31:08,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468376 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468382 2024-11-13T01:31:08,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468382) roll requested 2024-11-13T01:31:08,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468390 2024-11-13T01:31:08,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468382 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468390 2024-11-13T01:31:08,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468390) roll requested 2024-11-13T01:31:08,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468397 2024-11-13T01:31:08,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468397 2024-11-13T01:31:08,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,404 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468397) roll requested 2024-11-13T01:31:08,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468404 2024-11-13T01:31:08,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468404 2024-11-13T01:31:08,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,410 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468404) roll requested 2024-11-13T01:31:08,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468410 2024-11-13T01:31:08,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468404 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468410 2024-11-13T01:31:08,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468410) roll requested 2024-11-13T01:31:08,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468417 2024-11-13T01:31:08,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468410 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468417 2024-11-13T01:31:08,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468417) roll requested 2024-11-13T01:31:08,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468424 2024-11-13T01:31:08,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468424 2024-11-13T01:31:08,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468424) roll requested 2024-11-13T01:31:08,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468433 2024-11-13T01:31:08,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468433 2024-11-13T01:31:08,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,443 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468433) roll requested 2024-11-13T01:31:08,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468443 2024-11-13T01:31:08,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468443 2024-11-13T01:31:08,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,452 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468443) roll requested 2024-11-13T01:31:08,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468452 2024-11-13T01:31:08,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468452 2024-11-13T01:31:08,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468452) roll requested 2024-11-13T01:31:08,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468461 2024-11-13T01:31:08,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468461 2024-11-13T01:31:08,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468461) roll requested 2024-11-13T01:31:08,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468472 2024-11-13T01:31:08,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468472 2024-11-13T01:31:08,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468472) roll requested 2024-11-13T01:31:08,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468481 2024-11-13T01:31:08,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468481 2024-11-13T01:31:08,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468481) roll requested 2024-11-13T01:31:08,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468490 2024-11-13T01:31:08,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468490 2024-11-13T01:31:08,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468490) roll requested 2024-11-13T01:31:08,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468500 2024-11-13T01:31:08,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468500 2024-11-13T01:31:08,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,509 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468500) roll requested 2024-11-13T01:31:08,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468509 2024-11-13T01:31:08,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468509 2024-11-13T01:31:08,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468509) roll requested 2024-11-13T01:31:08,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468517 2024-11-13T01:31:08,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468517 2024-11-13T01:31:08,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468517) roll requested 2024-11-13T01:31:08,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468522 2024-11-13T01:31:08,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468522 2024-11-13T01:31:08,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,526 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468522) roll requested 2024-11-13T01:31:08,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468526 2024-11-13T01:31:08,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468522 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468526 2024-11-13T01:31:08,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468526) roll requested 2024-11-13T01:31:08,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468532 2024-11-13T01:31:08,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468532 2024-11-13T01:31:08,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468532) roll requested 2024-11-13T01:31:08,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468538 2024-11-13T01:31:08,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468538 2024-11-13T01:31:08,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,545 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468538) roll requested 2024-11-13T01:31:08,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468545 2024-11-13T01:31:08,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468545 2024-11-13T01:31:08,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468545) roll requested 2024-11-13T01:31:08,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468553 2024-11-13T01:31:08,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468553 2024-11-13T01:31:08,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468553) roll requested 2024-11-13T01:31:08,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468562 2024-11-13T01:31:08,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468562 2024-11-13T01:31:08,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,570 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468562) roll requested 2024-11-13T01:31:08,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468571 2024-11-13T01:31:08,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468571 2024-11-13T01:31:08,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468571) roll requested 2024-11-13T01:31:08,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468580 2024-11-13T01:31:08,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468571 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468580 2024-11-13T01:31:08,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,590 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468580) roll requested 2024-11-13T01:31:08,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468591 2024-11-13T01:31:08,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468591 2024-11-13T01:31:08,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468591) roll requested 2024-11-13T01:31:08,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468601 2024-11-13T01:31:08,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468601 2024-11-13T01:31:08,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468601) roll requested 2024-11-13T01:31:08,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468610 2024-11-13T01:31:08,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468610 2024-11-13T01:31:08,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,618 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468610) roll requested 2024-11-13T01:31:08,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468619 2024-11-13T01:31:08,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468619 2024-11-13T01:31:08,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468619) roll requested 2024-11-13T01:31:08,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468629 2024-11-13T01:31:08,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468629 2024-11-13T01:31:08,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468629) roll requested 2024-11-13T01:31:08,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468639 2024-11-13T01:31:08,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468639 2024-11-13T01:31:08,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,647 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468639) roll requested 2024-11-13T01:31:08,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468648 2024-11-13T01:31:08,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468639 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468648 2024-11-13T01:31:08,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468648) roll requested 2024-11-13T01:31:08,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468653 2024-11-13T01:31:08,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468648 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468653 2024-11-13T01:31:08,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468653) roll requested 2024-11-13T01:31:08,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468659 2024-11-13T01:31:08,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468659 2024-11-13T01:31:08,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468659) roll requested 2024-11-13T01:31:08,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468666 2024-11-13T01:31:08,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468666 2024-11-13T01:31:08,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468666) roll requested 2024-11-13T01:31:08,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468674 2024-11-13T01:31:08,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468666 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468674 2024-11-13T01:31:08,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468674) roll requested 2024-11-13T01:31:08,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468683 2024-11-13T01:31:08,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468683 2024-11-13T01:31:08,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,693 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468683) roll requested 2024-11-13T01:31:08,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468693 2024-11-13T01:31:08,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468693 2024-11-13T01:31:08,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468693) roll requested 2024-11-13T01:31:08,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468701 2024-11-13T01:31:08,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468701 2024-11-13T01:31:08,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468701) roll requested 2024-11-13T01:31:08,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468709 2024-11-13T01:31:08,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468709 2024-11-13T01:31:08,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468709) roll requested 2024-11-13T01:31:08,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468717 2024-11-13T01:31:08,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468709 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468717 2024-11-13T01:31:08,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468717) roll requested 2024-11-13T01:31:08,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468724 2024-11-13T01:31:08,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468724 2024-11-13T01:31:08,731 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,731 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468724) roll requested 2024-11-13T01:31:08,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468731 2024-11-13T01:31:08,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468731 2024-11-13T01:31:08,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468731) roll requested 2024-11-13T01:31:08,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468739 2024-11-13T01:31:08,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468739 2024-11-13T01:31:08,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468739) roll requested 2024-11-13T01:31:08,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468747 2024-11-13T01:31:08,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468747 2024-11-13T01:31:08,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468747) roll requested 2024-11-13T01:31:08,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468754 2024-11-13T01:31:08,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468754 2024-11-13T01:31:08,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468754) roll requested 2024-11-13T01:31:08,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468762 2024-11-13T01:31:08,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468754 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468762 2024-11-13T01:31:08,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468762) roll requested 2024-11-13T01:31:08,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468771 2024-11-13T01:31:08,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468771 2024-11-13T01:31:08,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468771) roll requested 2024-11-13T01:31:08,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468781 2024-11-13T01:31:08,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468771 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468781 2024-11-13T01:31:08,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,788 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468781) roll requested 2024-11-13T01:31:08,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468788 2024-11-13T01:31:08,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468788 2024-11-13T01:31:08,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,795 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468788) roll requested 2024-11-13T01:31:08,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468795 2024-11-13T01:31:08,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468788 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468795 2024-11-13T01:31:08,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468795) roll requested 2024-11-13T01:31:08,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468802 2024-11-13T01:31:08,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468795 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468802 2024-11-13T01:31:08,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468802) roll requested 2024-11-13T01:31:08,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468809 2024-11-13T01:31:08,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468809 2024-11-13T01:31:08,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468809) roll requested 2024-11-13T01:31:08,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468818 2024-11-13T01:31:08,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468818 2024-11-13T01:31:08,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468818) roll requested 2024-11-13T01:31:08,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468827 2024-11-13T01:31:08,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468827 2024-11-13T01:31:08,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468827) roll requested 2024-11-13T01:31:08,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468838 2024-11-13T01:31:08,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468838 2024-11-13T01:31:08,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468838) roll requested 2024-11-13T01:31:08,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468844 2024-11-13T01:31:08,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468844 2024-11-13T01:31:08,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468844) roll requested 2024-11-13T01:31:08,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468850 2024-11-13T01:31:08,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468850 2024-11-13T01:31:08,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468850) roll requested 2024-11-13T01:31:08,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468856 2024-11-13T01:31:08,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468856 2024-11-13T01:31:08,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468856) roll requested 2024-11-13T01:31:08,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468862 2024-11-13T01:31:08,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468856 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468862 2024-11-13T01:31:08,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,867 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468862) roll requested 2024-11-13T01:31:08,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468867 2024-11-13T01:31:08,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468867 2024-11-13T01:31:08,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468867) roll requested 2024-11-13T01:31:08,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468873 2024-11-13T01:31:08,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468867 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468873 2024-11-13T01:31:08,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468873) roll requested 2024-11-13T01:31:08,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468879 2024-11-13T01:31:08,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468879 2024-11-13T01:31:08,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468879) roll requested 2024-11-13T01:31:08,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468887 2024-11-13T01:31:08,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468887 2024-11-13T01:31:08,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,894 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468887) roll requested 2024-11-13T01:31:08,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468894 2024-11-13T01:31:08,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468887 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468894 2024-11-13T01:31:08,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,902 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468894) roll requested 2024-11-13T01:31:08,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468903 2024-11-13T01:31:08,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468894 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468903 2024-11-13T01:31:08,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468903) roll requested 2024-11-13T01:31:08,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468912 2024-11-13T01:31:08,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468912 2024-11-13T01:31:08,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468912) roll requested 2024-11-13T01:31:08,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468919 2024-11-13T01:31:08,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468919 2024-11-13T01:31:08,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,926 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468919) roll requested 2024-11-13T01:31:08,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468926 2024-11-13T01:31:08,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468926 2024-11-13T01:31:08,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,938 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468926) roll requested 2024-11-13T01:31:08,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468938 2024-11-13T01:31:08,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468938 2024-11-13T01:31:08,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468938) roll requested 2024-11-13T01:31:08,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468945 2024-11-13T01:31:08,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468938 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468945 2024-11-13T01:31:08,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,950 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468945) roll requested 2024-11-13T01:31:08,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468950 2024-11-13T01:31:08,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,955 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,955 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468950 2024-11-13T01:31:08,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,955 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468950) roll requested 2024-11-13T01:31:08,956 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468955 2024-11-13T01:31:08,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,961 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,961 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,961 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468955 2024-11-13T01:31:08,962 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,962 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,962 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468955) roll requested 2024-11-13T01:31:08,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468962 2024-11-13T01:31:08,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468955 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468962 2024-11-13T01:31:08,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468962) roll requested 2024-11-13T01:31:08,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468969 2024-11-13T01:31:08,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468962 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468969 2024-11-13T01:31:08,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468969) roll requested 2024-11-13T01:31:08,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468976 2024-11-13T01:31:08,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468976 2024-11-13T01:31:08,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468976) roll requested 2024-11-13T01:31:08,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468983 2024-11-13T01:31:08,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468983 2024-11-13T01:31:08,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:08,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:08,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468983) roll requested 2024-11-13T01:31:08,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461468991 2024-11-13T01:31:08,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:08,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468991 2024-11-13T01:31:09,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461468991) roll requested 2024-11-13T01:31:09,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469000 2024-11-13T01:31:09,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461468991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469000 2024-11-13T01:31:09,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469000) roll requested 2024-11-13T01:31:09,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469007 2024-11-13T01:31:09,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469007 2024-11-13T01:31:09,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469007) roll requested 2024-11-13T01:31:09,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469014 2024-11-13T01:31:09,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469014 2024-11-13T01:31:09,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469014) roll requested 2024-11-13T01:31:09,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469021 2024-11-13T01:31:09,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469021 2024-11-13T01:31:09,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469021) roll requested 2024-11-13T01:31:09,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469028 2024-11-13T01:31:09,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469028 2024-11-13T01:31:09,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469028) roll requested 2024-11-13T01:31:09,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469037 2024-11-13T01:31:09,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469037 2024-11-13T01:31:09,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,046 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469037) roll requested 2024-11-13T01:31:09,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469046 2024-11-13T01:31:09,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469046 2024-11-13T01:31:09,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469046) roll requested 2024-11-13T01:31:09,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469055 2024-11-13T01:31:09,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469055 2024-11-13T01:31:09,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,063 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469055) roll requested 2024-11-13T01:31:09,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469063 2024-11-13T01:31:09,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,070 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469063 2024-11-13T01:31:09,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469063) roll requested 2024-11-13T01:31:09,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469072 2024-11-13T01:31:09,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469063 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469072 2024-11-13T01:31:09,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,082 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469072) roll requested 2024-11-13T01:31:09,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469082 2024-11-13T01:31:09,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469082 2024-11-13T01:31:09,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469082) roll requested 2024-11-13T01:31:09,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469093 2024-11-13T01:31:09,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469082 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469093 2024-11-13T01:31:09,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469093) roll requested 2024-11-13T01:31:09,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469099 2024-11-13T01:31:09,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469099 2024-11-13T01:31:09,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469099) roll requested 2024-11-13T01:31:09,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469105 2024-11-13T01:31:09,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469099 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469105 2024-11-13T01:31:09,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469105) roll requested 2024-11-13T01:31:09,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469112 2024-11-13T01:31:09,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469112 2024-11-13T01:31:09,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469112) roll requested 2024-11-13T01:31:09,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469119 2024-11-13T01:31:09,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469119 2024-11-13T01:31:09,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469119) roll requested 2024-11-13T01:31:09,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469128 2024-11-13T01:31:09,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469128 2024-11-13T01:31:09,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,138 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469128) roll requested 2024-11-13T01:31:09,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469138 2024-11-13T01:31:09,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469138 2024-11-13T01:31:09,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469138) roll requested 2024-11-13T01:31:09,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469148 2024-11-13T01:31:09,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469148 2024-11-13T01:31:09,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469148) roll requested 2024-11-13T01:31:09,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469153 2024-11-13T01:31:09,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469153 2024-11-13T01:31:09,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469153) roll requested 2024-11-13T01:31:09,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469158 2024-11-13T01:31:09,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469158 2024-11-13T01:31:09,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469158) roll requested 2024-11-13T01:31:09,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469165 2024-11-13T01:31:09,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469165 2024-11-13T01:31:09,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469165) roll requested 2024-11-13T01:31:09,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469173 2024-11-13T01:31:09,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,181 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469173 2024-11-13T01:31:09,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469173) roll requested 2024-11-13T01:31:09,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469182 2024-11-13T01:31:09,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469182 2024-11-13T01:31:09,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469182) roll requested 2024-11-13T01:31:09,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469188 2024-11-13T01:31:09,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469182 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469188 2024-11-13T01:31:09,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469188) roll requested 2024-11-13T01:31:09,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469196 2024-11-13T01:31:09,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469196 2024-11-13T01:31:09,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469196) roll requested 2024-11-13T01:31:09,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469203 2024-11-13T01:31:09,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469203 2024-11-13T01:31:09,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469203) roll requested 2024-11-13T01:31:09,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469211 2024-11-13T01:31:09,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469211 2024-11-13T01:31:09,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469211) roll requested 2024-11-13T01:31:09,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469216 2024-11-13T01:31:09,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469211 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469216 2024-11-13T01:31:09,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469216) roll requested 2024-11-13T01:31:09,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469223 2024-11-13T01:31:09,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469223 2024-11-13T01:31:09,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469223) roll requested 2024-11-13T01:31:09,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469230 2024-11-13T01:31:09,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469230 2024-11-13T01:31:09,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469230) roll requested 2024-11-13T01:31:09,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469236 2024-11-13T01:31:09,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469236 2024-11-13T01:31:09,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:09,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469236) roll requested 2024-11-13T01:31:09,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469245 2024-11-13T01:31:09,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469245 2024-11-13T01:31:09,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469245) roll requested 2024-11-13T01:31:09,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469252 2024-11-13T01:31:09,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469252 2024-11-13T01:31:09,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469252) roll requested 2024-11-13T01:31:09,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469259 2024-11-13T01:31:09,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469259 2024-11-13T01:31:09,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469259) roll requested 2024-11-13T01:31:09,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469269 2024-11-13T01:31:09,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469269 2024-11-13T01:31:09,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469269) roll requested 2024-11-13T01:31:09,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469278 2024-11-13T01:31:09,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469278 2024-11-13T01:31:09,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469278) roll requested 2024-11-13T01:31:09,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469283 2024-11-13T01:31:09,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469283 2024-11-13T01:31:09,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469283) roll requested 2024-11-13T01:31:09,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469289 2024-11-13T01:31:09,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,294 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,294 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,294 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469289 2024-11-13T01:31:09,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469289) roll requested 2024-11-13T01:31:09,295 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469295 2024-11-13T01:31:09,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469295 2024-11-13T01:31:09,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469295) roll requested 2024-11-13T01:31:09,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469303 2024-11-13T01:31:09,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469303 2024-11-13T01:31:09,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469303) roll requested 2024-11-13T01:31:09,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469311 2024-11-13T01:31:09,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469311 2024-11-13T01:31:09,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,319 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469311) roll requested 2024-11-13T01:31:09,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469320 2024-11-13T01:31:09,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469320 2024-11-13T01:31:09,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469320) roll requested 2024-11-13T01:31:09,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469325 2024-11-13T01:31:09,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469325 2024-11-13T01:31:09,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469325) roll requested 2024-11-13T01:31:09,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469333 2024-11-13T01:31:09,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469333 2024-11-13T01:31:09,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469333) roll requested 2024-11-13T01:31:09,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469338 2024-11-13T01:31:09,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469338 2024-11-13T01:31:09,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469338) roll requested 2024-11-13T01:31:09,344 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469344 2024-11-13T01:31:09,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469344 2024-11-13T01:31:09,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469344) roll requested 2024-11-13T01:31:09,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469351 2024-11-13T01:31:09,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,357 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469344 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469351 2024-11-13T01:31:09,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,358 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469351) roll requested 2024-11-13T01:31:09,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469358 2024-11-13T01:31:09,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469358 2024-11-13T01:31:09,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469358) roll requested 2024-11-13T01:31:09,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469366 2024-11-13T01:31:09,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469366 2024-11-13T01:31:09,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469358 is not closed yet, will try archiving it next time 2024-11-13T01:31:09,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469366) roll requested 2024-11-13T01:31:09,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469376 2024-11-13T01:31:09,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469376 2024-11-13T01:31:09,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469376) roll requested 2024-11-13T01:31:09,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469572 2024-11-13T01:31:09,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469572 2024-11-13T01:31:09,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469572) roll requested 2024-11-13T01:31:09,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469581 2024-11-13T01:31:09,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469581 2024-11-13T01:31:09,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469581) roll requested 2024-11-13T01:31:09,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469588 2024-11-13T01:31:09,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469588 2024-11-13T01:31:09,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469588) roll requested 2024-11-13T01:31:09,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469596 2024-11-13T01:31:09,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,603 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,603 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469588 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469596 2024-11-13T01:31:09,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469596) roll requested 2024-11-13T01:31:09,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469605 2024-11-13T01:31:09,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469605 2024-11-13T01:31:09,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469605) roll requested 2024-11-13T01:31:09,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469614 2024-11-13T01:31:09,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469605 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469614 2024-11-13T01:31:09,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469614) roll requested 2024-11-13T01:31:09,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469627 2024-11-13T01:31:09,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469627 2024-11-13T01:31:09,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469627) roll requested 2024-11-13T01:31:09,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469637 2024-11-13T01:31:09,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469637 2024-11-13T01:31:09,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469637) roll requested 2024-11-13T01:31:09,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469643 2024-11-13T01:31:09,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469643 2024-11-13T01:31:09,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469643) roll requested 2024-11-13T01:31:09,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469648 2024-11-13T01:31:09,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469648 2024-11-13T01:31:09,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469648) roll requested 2024-11-13T01:31:09,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469655 2024-11-13T01:31:09,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469655 2024-11-13T01:31:09,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469655) roll requested 2024-11-13T01:31:09,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469662 2024-11-13T01:31:09,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469662 2024-11-13T01:31:09,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469662) roll requested 2024-11-13T01:31:09,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469670 2024-11-13T01:31:09,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469662 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469670 2024-11-13T01:31:09,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469670) roll requested 2024-11-13T01:31:09,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469678 2024-11-13T01:31:09,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469678 2024-11-13T01:31:09,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469678) roll requested 2024-11-13T01:31:09,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469687 2024-11-13T01:31:09,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469687 2024-11-13T01:31:09,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469687) roll requested 2024-11-13T01:31:09,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469696 2024-11-13T01:31:09,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469696 2024-11-13T01:31:09,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469696) roll requested 2024-11-13T01:31:09,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469705 2024-11-13T01:31:09,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469705 2024-11-13T01:31:09,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469705) roll requested 2024-11-13T01:31:09,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469713 2024-11-13T01:31:09,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469713 2024-11-13T01:31:09,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469713) roll requested 2024-11-13T01:31:09,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469719 2024-11-13T01:31:09,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469719 2024-11-13T01:31:09,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469719) roll requested 2024-11-13T01:31:09,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469727 2024-11-13T01:31:09,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469727 2024-11-13T01:31:09,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469727) roll requested 2024-11-13T01:31:09,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469735 2024-11-13T01:31:09,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469735 2024-11-13T01:31:09,746 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,746 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,746 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,746 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469735) roll requested 2024-11-13T01:31:09,746 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469746 2024-11-13T01:31:09,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469735 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469746 2024-11-13T01:31:09,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469746) roll requested 2024-11-13T01:31:09,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469755 2024-11-13T01:31:09,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469746 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469755 2024-11-13T01:31:09,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469755) roll requested 2024-11-13T01:31:09,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469763 2024-11-13T01:31:09,770 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,770 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469755 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469763 2024-11-13T01:31:09,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469763) roll requested 2024-11-13T01:31:09,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469772 2024-11-13T01:31:09,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469763 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469772 2024-11-13T01:31:09,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469772) roll requested 2024-11-13T01:31:09,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469782 2024-11-13T01:31:09,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469782 2024-11-13T01:31:09,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469782) roll requested 2024-11-13T01:31:09,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469801 2024-11-13T01:31:09,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469801 2024-11-13T01:31:09,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469801) roll requested 2024-11-13T01:31:09,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469812 2024-11-13T01:31:09,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469812 2024-11-13T01:31:09,819 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,819 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469812) roll requested 2024-11-13T01:31:09,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469819 2024-11-13T01:31:09,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469819 2024-11-13T01:31:09,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469819) roll requested 2024-11-13T01:31:09,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469825 2024-11-13T01:31:09,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469825 2024-11-13T01:31:09,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469825) roll requested 2024-11-13T01:31:09,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469833 2024-11-13T01:31:09,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469833 2024-11-13T01:31:09,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469833) roll requested 2024-11-13T01:31:09,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469840 2024-11-13T01:31:09,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469840 2024-11-13T01:31:09,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469840) roll requested 2024-11-13T01:31:09,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469846 2024-11-13T01:31:09,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469846 2024-11-13T01:31:09,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469846) roll requested 2024-11-13T01:31:09,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469851 2024-11-13T01:31:09,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469851 2024-11-13T01:31:09,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469851) roll requested 2024-11-13T01:31:09,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469857 2024-11-13T01:31:09,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469857 2024-11-13T01:31:09,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469857) roll requested 2024-11-13T01:31:09,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469863 2024-11-13T01:31:09,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469863 2024-11-13T01:31:09,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,871 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469863) roll requested 2024-11-13T01:31:09,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469871 2024-11-13T01:31:09,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469871 2024-11-13T01:31:09,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469871) roll requested 2024-11-13T01:31:09,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469883 2024-11-13T01:31:09,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469883 2024-11-13T01:31:09,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469883) roll requested 2024-11-13T01:31:09,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469889 2024-11-13T01:31:09,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469889 2024-11-13T01:31:09,894 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469889) roll requested 2024-11-13T01:31:09,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469894 2024-11-13T01:31:09,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469894 2024-11-13T01:31:09,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469894) roll requested 2024-11-13T01:31:09,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469901 2024-11-13T01:31:09,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469901 2024-11-13T01:31:09,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469901) roll requested 2024-11-13T01:31:09,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469906 2024-11-13T01:31:09,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469906 2024-11-13T01:31:09,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469906) roll requested 2024-11-13T01:31:09,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469913 2024-11-13T01:31:09,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469913 2024-11-13T01:31:09,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469913) roll requested 2024-11-13T01:31:09,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469919 2024-11-13T01:31:09,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469919 2024-11-13T01:31:09,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469919) roll requested 2024-11-13T01:31:09,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469924 2024-11-13T01:31:09,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469924 2024-11-13T01:31:09,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469924) roll requested 2024-11-13T01:31:09,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469930 2024-11-13T01:31:09,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469930 2024-11-13T01:31:09,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469930) roll requested 2024-11-13T01:31:09,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469936 2024-11-13T01:31:09,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469936 2024-11-13T01:31:09,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469936) roll requested 2024-11-13T01:31:09,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469944 2024-11-13T01:31:09,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469944 2024-11-13T01:31:09,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469944) roll requested 2024-11-13T01:31:09,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469951 2024-11-13T01:31:09,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469944 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469951 2024-11-13T01:31:09,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,959 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469951) roll requested 2024-11-13T01:31:09,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469959 2024-11-13T01:31:09,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469959 2024-11-13T01:31:09,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469959) roll requested 2024-11-13T01:31:09,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469968 2024-11-13T01:31:09,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469968 2024-11-13T01:31:09,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469968) roll requested 2024-11-13T01:31:09,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469975 2024-11-13T01:31:09,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469975 2024-11-13T01:31:09,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469975) roll requested 2024-11-13T01:31:09,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469983 2024-11-13T01:31:09,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:09,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469983 2024-11-13T01:31:09,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:09,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:09,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:09,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:09,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469983) roll requested 2024-11-13T01:31:09,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461469992 2024-11-13T01:31:10,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469992 2024-11-13T01:31:10,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461469992) roll requested 2024-11-13T01:31:10,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470002 2024-11-13T01:31:10,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461469992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470002 2024-11-13T01:31:10,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,008 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470002) roll requested 2024-11-13T01:31:10,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470008 2024-11-13T01:31:10,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470008 2024-11-13T01:31:10,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470008) roll requested 2024-11-13T01:31:10,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470015 2024-11-13T01:31:10,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470008 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470015 2024-11-13T01:31:10,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470015) roll requested 2024-11-13T01:31:10,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470020 2024-11-13T01:31:10,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470020 2024-11-13T01:31:10,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470020) roll requested 2024-11-13T01:31:10,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470026 2024-11-13T01:31:10,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470026 2024-11-13T01:31:10,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470026) roll requested 2024-11-13T01:31:10,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470032 2024-11-13T01:31:10,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470032 2024-11-13T01:31:10,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470032) roll requested 2024-11-13T01:31:10,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470038 2024-11-13T01:31:10,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470038 2024-11-13T01:31:10,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470038) roll requested 2024-11-13T01:31:10,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470043 2024-11-13T01:31:10,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470043 2024-11-13T01:31:10,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470043) roll requested 2024-11-13T01:31:10,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470049 2024-11-13T01:31:10,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470049 2024-11-13T01:31:10,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470049) roll requested 2024-11-13T01:31:10,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470055 2024-11-13T01:31:10,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470055 2024-11-13T01:31:10,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470055) roll requested 2024-11-13T01:31:10,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470060 2024-11-13T01:31:10,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470060 2024-11-13T01:31:10,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470060) roll requested 2024-11-13T01:31:10,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470066 2024-11-13T01:31:10,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470066 2024-11-13T01:31:10,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,072 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470066) roll requested 2024-11-13T01:31:10,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470072 2024-11-13T01:31:10,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470072 2024-11-13T01:31:10,077 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,077 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470072) roll requested 2024-11-13T01:31:10,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470078 2024-11-13T01:31:10,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470072 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470078 2024-11-13T01:31:10,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,083 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,083 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,083 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470078) roll requested 2024-11-13T01:31:10,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470084 2024-11-13T01:31:10,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470078 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470084 2024-11-13T01:31:10,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470084) roll requested 2024-11-13T01:31:10,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470092 2024-11-13T01:31:10,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470092 2024-11-13T01:31:10,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470092) roll requested 2024-11-13T01:31:10,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470098 2024-11-13T01:31:10,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470098 2024-11-13T01:31:10,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470098) roll requested 2024-11-13T01:31:10,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470105 2024-11-13T01:31:10,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470105 2024-11-13T01:31:10,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470105) roll requested 2024-11-13T01:31:10,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470110 2024-11-13T01:31:10,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470110 2024-11-13T01:31:10,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470110) roll requested 2024-11-13T01:31:10,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470116 2024-11-13T01:31:10,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470116 2024-11-13T01:31:10,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470116) roll requested 2024-11-13T01:31:10,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470121 2024-11-13T01:31:10,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470121 2024-11-13T01:31:10,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470121) roll requested 2024-11-13T01:31:10,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470126 2024-11-13T01:31:10,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470126 2024-11-13T01:31:10,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,131 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,131 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470126) roll requested 2024-11-13T01:31:10,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470132 2024-11-13T01:31:10,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470132 2024-11-13T01:31:10,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470132) roll requested 2024-11-13T01:31:10,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470137 2024-11-13T01:31:10,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470137 2024-11-13T01:31:10,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470137) roll requested 2024-11-13T01:31:10,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470144 2024-11-13T01:31:10,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470144 2024-11-13T01:31:10,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470144) roll requested 2024-11-13T01:31:10,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470151 2024-11-13T01:31:10,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470151 2024-11-13T01:31:10,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470151) roll requested 2024-11-13T01:31:10,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470160 2024-11-13T01:31:10,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470160 2024-11-13T01:31:10,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470160) roll requested 2024-11-13T01:31:10,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470169 2024-11-13T01:31:10,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470169 2024-11-13T01:31:10,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470169) roll requested 2024-11-13T01:31:10,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470177 2024-11-13T01:31:10,183 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,183 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470177 2024-11-13T01:31:10,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470177) roll requested 2024-11-13T01:31:10,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470185 2024-11-13T01:31:10,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470185 2024-11-13T01:31:10,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,194 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,194 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470185) roll requested 2024-11-13T01:31:10,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470195 2024-11-13T01:31:10,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470195 2024-11-13T01:31:10,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,204 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470195) roll requested 2024-11-13T01:31:10,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470204 2024-11-13T01:31:10,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470204 2024-11-13T01:31:10,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470204) roll requested 2024-11-13T01:31:10,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470213 2024-11-13T01:31:10,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470213 2024-11-13T01:31:10,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,220 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470213) roll requested 2024-11-13T01:31:10,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470220 2024-11-13T01:31:10,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470220 2024-11-13T01:31:10,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470220) roll requested 2024-11-13T01:31:10,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470230 2024-11-13T01:31:10,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470230 2024-11-13T01:31:10,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470230) roll requested 2024-11-13T01:31:10,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470237 2024-11-13T01:31:10,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470237 2024-11-13T01:31:10,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470237) roll requested 2024-11-13T01:31:10,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470246 2024-11-13T01:31:10,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470246 2024-11-13T01:31:10,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470246) roll requested 2024-11-13T01:31:10,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470252 2024-11-13T01:31:10,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470246 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470252 2024-11-13T01:31:10,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,259 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470252) roll requested 2024-11-13T01:31:10,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470260 2024-11-13T01:31:10,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470260 2024-11-13T01:31:10,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470260) roll requested 2024-11-13T01:31:10,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470269 2024-11-13T01:31:10,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470269 2024-11-13T01:31:10,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,275 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,275 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,275 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470269) roll requested 2024-11-13T01:31:10,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470275 2024-11-13T01:31:10,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470275 2024-11-13T01:31:10,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470275) roll requested 2024-11-13T01:31:10,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470285 2024-11-13T01:31:10,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470275 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470285 2024-11-13T01:31:10,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470285) roll requested 2024-11-13T01:31:10,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470290 2024-11-13T01:31:10,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470290 2024-11-13T01:31:10,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470290) roll requested 2024-11-13T01:31:10,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470296 2024-11-13T01:31:10,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470296 2024-11-13T01:31:10,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470296) roll requested 2024-11-13T01:31:10,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470305 2024-11-13T01:31:10,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,310 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470305 2024-11-13T01:31:10,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470305) roll requested 2024-11-13T01:31:10,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470311 2024-11-13T01:31:10,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470311 2024-11-13T01:31:10,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470311) roll requested 2024-11-13T01:31:10,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470317 2024-11-13T01:31:10,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470317 2024-11-13T01:31:10,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470317) roll requested 2024-11-13T01:31:10,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470322 2024-11-13T01:31:10,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470322 2024-11-13T01:31:10,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470322) roll requested 2024-11-13T01:31:10,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470329 2024-11-13T01:31:10,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470329 2024-11-13T01:31:10,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470329) roll requested 2024-11-13T01:31:10,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470334 2024-11-13T01:31:10,338 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,338 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470334 2024-11-13T01:31:10,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470334) roll requested 2024-11-13T01:31:10,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470340 2024-11-13T01:31:10,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470340 2024-11-13T01:31:10,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470340) roll requested 2024-11-13T01:31:10,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470345 2024-11-13T01:31:10,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470340 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470345 2024-11-13T01:31:10,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470345) roll requested 2024-11-13T01:31:10,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470351 2024-11-13T01:31:10,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470351 2024-11-13T01:31:10,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470351) roll requested 2024-11-13T01:31:10,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470357 2024-11-13T01:31:10,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470357 2024-11-13T01:31:10,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,363 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,363 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470357) roll requested 2024-11-13T01:31:10,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470363 2024-11-13T01:31:10,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470363 2024-11-13T01:31:10,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470363) roll requested 2024-11-13T01:31:10,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470368 2024-11-13T01:31:10,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470363 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470368 2024-11-13T01:31:10,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470368) roll requested 2024-11-13T01:31:10,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470375 2024-11-13T01:31:10,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470375 2024-11-13T01:31:10,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470375) roll requested 2024-11-13T01:31:10,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470380 2024-11-13T01:31:10,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470380 2024-11-13T01:31:10,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470380) roll requested 2024-11-13T01:31:10,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470386 2024-11-13T01:31:10,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470386 2024-11-13T01:31:10,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,392 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470386) roll requested 2024-11-13T01:31:10,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470393 2024-11-13T01:31:10,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470393 2024-11-13T01:31:10,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470393) roll requested 2024-11-13T01:31:10,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470397 2024-11-13T01:31:10,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470393 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470397 2024-11-13T01:31:10,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470397) roll requested 2024-11-13T01:31:10,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470402 2024-11-13T01:31:10,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470402 2024-11-13T01:31:10,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470402) roll requested 2024-11-13T01:31:10,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470409 2024-11-13T01:31:10,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470409 2024-11-13T01:31:10,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470409) roll requested 2024-11-13T01:31:10,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470415 2024-11-13T01:31:10,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470415 2024-11-13T01:31:10,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470415) roll requested 2024-11-13T01:31:10,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470423 2024-11-13T01:31:10,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470423 2024-11-13T01:31:10,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470423) roll requested 2024-11-13T01:31:10,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470431 2024-11-13T01:31:10,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470431 2024-11-13T01:31:10,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470431) roll requested 2024-11-13T01:31:10,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470440 2024-11-13T01:31:10,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470440 2024-11-13T01:31:10,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470440) roll requested 2024-11-13T01:31:10,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470450 2024-11-13T01:31:10,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470450 2024-11-13T01:31:10,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470450) roll requested 2024-11-13T01:31:10,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470458 2024-11-13T01:31:10,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470458 2024-11-13T01:31:10,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470458) roll requested 2024-11-13T01:31:10,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470467 2024-11-13T01:31:10,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470467 2024-11-13T01:31:10,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470467) roll requested 2024-11-13T01:31:10,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470476 2024-11-13T01:31:10,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470476 2024-11-13T01:31:10,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470476) roll requested 2024-11-13T01:31:10,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470485 2024-11-13T01:31:10,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470485 2024-11-13T01:31:10,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470485) roll requested 2024-11-13T01:31:10,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470492 2024-11-13T01:31:10,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470492 2024-11-13T01:31:10,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470492) roll requested 2024-11-13T01:31:10,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470499 2024-11-13T01:31:10,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470499 2024-11-13T01:31:10,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470499) roll requested 2024-11-13T01:31:10,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470506 2024-11-13T01:31:10,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470506 2024-11-13T01:31:10,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470506) roll requested 2024-11-13T01:31:10,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470514 2024-11-13T01:31:10,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470514 2024-11-13T01:31:10,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470514) roll requested 2024-11-13T01:31:10,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470524 2024-11-13T01:31:10,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470524 2024-11-13T01:31:10,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470524) roll requested 2024-11-13T01:31:10,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470530 2024-11-13T01:31:10,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470530 2024-11-13T01:31:10,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470530) roll requested 2024-11-13T01:31:10,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470536 2024-11-13T01:31:10,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470536 2024-11-13T01:31:10,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,542 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,542 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470536) roll requested 2024-11-13T01:31:10,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470542 2024-11-13T01:31:10,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470542 2024-11-13T01:31:10,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470542) roll requested 2024-11-13T01:31:10,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470549 2024-11-13T01:31:10,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470542 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470549 2024-11-13T01:31:10,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470549) roll requested 2024-11-13T01:31:10,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470557 2024-11-13T01:31:10,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470557 2024-11-13T01:31:10,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470557) roll requested 2024-11-13T01:31:10,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470564 2024-11-13T01:31:10,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470564 2024-11-13T01:31:10,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470564) roll requested 2024-11-13T01:31:10,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470572 2024-11-13T01:31:10,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470572 2024-11-13T01:31:10,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,581 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470572) roll requested 2024-11-13T01:31:10,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470581 2024-11-13T01:31:10,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470572 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470581 2024-11-13T01:31:10,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470581) roll requested 2024-11-13T01:31:10,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470591 2024-11-13T01:31:10,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470591 2024-11-13T01:31:10,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470591) roll requested 2024-11-13T01:31:10,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470600 2024-11-13T01:31:10,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470600 2024-11-13T01:31:10,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470600) roll requested 2024-11-13T01:31:10,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470607 2024-11-13T01:31:10,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470607 2024-11-13T01:31:10,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470607) roll requested 2024-11-13T01:31:10,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470614 2024-11-13T01:31:10,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470614 2024-11-13T01:31:10,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470614) roll requested 2024-11-13T01:31:10,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470625 2024-11-13T01:31:10,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470625 2024-11-13T01:31:10,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470625) roll requested 2024-11-13T01:31:10,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470630 2024-11-13T01:31:10,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470630 2024-11-13T01:31:10,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470630) roll requested 2024-11-13T01:31:10,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470638 2024-11-13T01:31:10,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470638 2024-11-13T01:31:10,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470638) roll requested 2024-11-13T01:31:10,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470648 2024-11-13T01:31:10,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470648 2024-11-13T01:31:10,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470648) roll requested 2024-11-13T01:31:10,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470654 2024-11-13T01:31:10,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470654 2024-11-13T01:31:10,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470654) roll requested 2024-11-13T01:31:10,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470659 2024-11-13T01:31:10,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470659 2024-11-13T01:31:10,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,665 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,665 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470659) roll requested 2024-11-13T01:31:10,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470666 2024-11-13T01:31:10,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470666 2024-11-13T01:31:10,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470666) roll requested 2024-11-13T01:31:10,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470674 2024-11-13T01:31:10,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470674 2024-11-13T01:31:10,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,682 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470674) roll requested 2024-11-13T01:31:10,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470683 2024-11-13T01:31:10,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470674 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470683 2024-11-13T01:31:10,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470683) roll requested 2024-11-13T01:31:10,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470691 2024-11-13T01:31:10,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470691 2024-11-13T01:31:10,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470691) roll requested 2024-11-13T01:31:10,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470697 2024-11-13T01:31:10,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470697 2024-11-13T01:31:10,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470697) roll requested 2024-11-13T01:31:10,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470703 2024-11-13T01:31:10,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470703 2024-11-13T01:31:10,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470703) roll requested 2024-11-13T01:31:10,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470708 2024-11-13T01:31:10,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470708 2024-11-13T01:31:10,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470708) roll requested 2024-11-13T01:31:10,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470714 2024-11-13T01:31:10,719 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,719 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470708 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470714 2024-11-13T01:31:10,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470714) roll requested 2024-11-13T01:31:10,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470720 2024-11-13T01:31:10,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470720 2024-11-13T01:31:10,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,728 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470720) roll requested 2024-11-13T01:31:10,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470729 2024-11-13T01:31:10,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470729 2024-11-13T01:31:10,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470729) roll requested 2024-11-13T01:31:10,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470734 2024-11-13T01:31:10,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470734 2024-11-13T01:31:10,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470734) roll requested 2024-11-13T01:31:10,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470741 2024-11-13T01:31:10,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470741 2024-11-13T01:31:10,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470741) roll requested 2024-11-13T01:31:10,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470748 2024-11-13T01:31:10,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470748 2024-11-13T01:31:10,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,755 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,755 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,755 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470748) roll requested 2024-11-13T01:31:10,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470756 2024-11-13T01:31:10,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470756 2024-11-13T01:31:10,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470756) roll requested 2024-11-13T01:31:10,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470761 2024-11-13T01:31:10,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470761 2024-11-13T01:31:10,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470761) roll requested 2024-11-13T01:31:10,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470768 2024-11-13T01:31:10,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470768 2024-11-13T01:31:10,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470768) roll requested 2024-11-13T01:31:10,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470774 2024-11-13T01:31:10,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470774 2024-11-13T01:31:10,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470774) roll requested 2024-11-13T01:31:10,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470780 2024-11-13T01:31:10,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470780 2024-11-13T01:31:10,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,785 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,785 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470780) roll requested 2024-11-13T01:31:10,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470785 2024-11-13T01:31:10,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470785 2024-11-13T01:31:10,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470785) roll requested 2024-11-13T01:31:10,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470791 2024-11-13T01:31:10,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470791 2024-11-13T01:31:10,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470791) roll requested 2024-11-13T01:31:10,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470797 2024-11-13T01:31:10,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470797 2024-11-13T01:31:10,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470797) roll requested 2024-11-13T01:31:10,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470802 2024-11-13T01:31:10,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470802 2024-11-13T01:31:10,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,809 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470802) roll requested 2024-11-13T01:31:10,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470809 2024-11-13T01:31:10,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470809 2024-11-13T01:31:10,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470809) roll requested 2024-11-13T01:31:10,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470815 2024-11-13T01:31:10,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470809 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470815 2024-11-13T01:31:10,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470815) roll requested 2024-11-13T01:31:10,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470821 2024-11-13T01:31:10,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470821 2024-11-13T01:31:10,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,827 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,827 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470821) roll requested 2024-11-13T01:31:10,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470827 2024-11-13T01:31:10,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470827 2024-11-13T01:31:10,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,832 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470827) roll requested 2024-11-13T01:31:10,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470832 2024-11-13T01:31:10,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470827 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470832 2024-11-13T01:31:10,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,838 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470832) roll requested 2024-11-13T01:31:10,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470839 2024-11-13T01:31:10,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470839 2024-11-13T01:31:10,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,845 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470839) roll requested 2024-11-13T01:31:10,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470845 2024-11-13T01:31:10,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470845 2024-11-13T01:31:10,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470845) roll requested 2024-11-13T01:31:10,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470851 2024-11-13T01:31:10,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470851 2024-11-13T01:31:10,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470851) roll requested 2024-11-13T01:31:10,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470856 2024-11-13T01:31:10,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470856 2024-11-13T01:31:10,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470856) roll requested 2024-11-13T01:31:10,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470862 2024-11-13T01:31:10,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470862 2024-11-13T01:31:10,867 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,867 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,867 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470862) roll requested 2024-11-13T01:31:10,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470867 2024-11-13T01:31:10,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470867 2024-11-13T01:31:10,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470867) roll requested 2024-11-13T01:31:10,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470873 2024-11-13T01:31:10,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470873 2024-11-13T01:31:10,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470873) roll requested 2024-11-13T01:31:10,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470879 2024-11-13T01:31:10,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470879 2024-11-13T01:31:10,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470879) roll requested 2024-11-13T01:31:10,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470884 2024-11-13T01:31:10,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470884 2024-11-13T01:31:10,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470884) roll requested 2024-11-13T01:31:10,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470893 2024-11-13T01:31:10,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470893 2024-11-13T01:31:10,898 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,898 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,898 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,898 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470893) roll requested 2024-11-13T01:31:10,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470898 2024-11-13T01:31:10,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470898 2024-11-13T01:31:10,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470898) roll requested 2024-11-13T01:31:10,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470904 2024-11-13T01:31:10,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470904 2024-11-13T01:31:10,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,910 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470904) roll requested 2024-11-13T01:31:10,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470910 2024-11-13T01:31:10,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470910 2024-11-13T01:31:10,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470910) roll requested 2024-11-13T01:31:10,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470917 2024-11-13T01:31:10,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470910 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470917 2024-11-13T01:31:10,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470917) roll requested 2024-11-13T01:31:10,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470924 2024-11-13T01:31:10,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470924 2024-11-13T01:31:10,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470924) roll requested 2024-11-13T01:31:10,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470932 2024-11-13T01:31:10,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470932 2024-11-13T01:31:10,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470932) roll requested 2024-11-13T01:31:10,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470941 2024-11-13T01:31:10,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470941 2024-11-13T01:31:10,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470941) roll requested 2024-11-13T01:31:10,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470947 2024-11-13T01:31:10,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470947 2024-11-13T01:31:10,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470947) roll requested 2024-11-13T01:31:10,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470954 2024-11-13T01:31:10,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470954 2024-11-13T01:31:10,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,963 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,963 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470954) roll requested 2024-11-13T01:31:10,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470964 2024-11-13T01:31:10,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470964 2024-11-13T01:31:10,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470964) roll requested 2024-11-13T01:31:10,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470970 2024-11-13T01:31:10,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470970 2024-11-13T01:31:10,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470970) roll requested 2024-11-13T01:31:10,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470975 2024-11-13T01:31:10,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470975 2024-11-13T01:31:10,980 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470975) roll requested 2024-11-13T01:31:10,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470981 2024-11-13T01:31:10,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470981 2024-11-13T01:31:10,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470981) roll requested 2024-11-13T01:31:10,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470988 2024-11-13T01:31:10,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:10,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470988 2024-11-13T01:31:10,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:10,996 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:10,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:10,996 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:10,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470988) roll requested 2024-11-13T01:31:10,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461470996 2024-11-13T01:31:11,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470996 2024-11-13T01:31:11,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461470996) roll requested 2024-11-13T01:31:11,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471003 2024-11-13T01:31:11,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461470996 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471003 2024-11-13T01:31:11,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471003) roll requested 2024-11-13T01:31:11,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471011 2024-11-13T01:31:11,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471011 2024-11-13T01:31:11,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,022 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471011) roll requested 2024-11-13T01:31:11,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471022 2024-11-13T01:31:11,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471022 2024-11-13T01:31:11,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471022) roll requested 2024-11-13T01:31:11,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471028 2024-11-13T01:31:11,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471022 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471028 2024-11-13T01:31:11,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471028) roll requested 2024-11-13T01:31:11,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471033 2024-11-13T01:31:11,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471033 2024-11-13T01:31:11,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471033) roll requested 2024-11-13T01:31:11,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471038 2024-11-13T01:31:11,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471038 2024-11-13T01:31:11,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,045 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471038) roll requested 2024-11-13T01:31:11,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471045 2024-11-13T01:31:11,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471045 2024-11-13T01:31:11,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471045) roll requested 2024-11-13T01:31:11,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471052 2024-11-13T01:31:11,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471052 2024-11-13T01:31:11,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471052) roll requested 2024-11-13T01:31:11,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471058 2024-11-13T01:31:11,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471058 2024-11-13T01:31:11,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471058) roll requested 2024-11-13T01:31:11,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471067 2024-11-13T01:31:11,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471067 2024-11-13T01:31:11,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471067) roll requested 2024-11-13T01:31:11,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471076 2024-11-13T01:31:11,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471076 2024-11-13T01:31:11,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471076) roll requested 2024-11-13T01:31:11,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471085 2024-11-13T01:31:11,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471085 2024-11-13T01:31:11,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,095 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471085) roll requested 2024-11-13T01:31:11,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471096 2024-11-13T01:31:11,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471096 2024-11-13T01:31:11,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471096) roll requested 2024-11-13T01:31:11,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471104 2024-11-13T01:31:11,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471104 2024-11-13T01:31:11,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471104) roll requested 2024-11-13T01:31:11,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471112 2024-11-13T01:31:11,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471112 2024-11-13T01:31:11,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471112) roll requested 2024-11-13T01:31:11,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471118 2024-11-13T01:31:11,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471118 2024-11-13T01:31:11,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471118) roll requested 2024-11-13T01:31:11,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471126 2024-11-13T01:31:11,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471126 2024-11-13T01:31:11,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,133 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,133 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471126) roll requested 2024-11-13T01:31:11,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471133 2024-11-13T01:31:11,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,139 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,139 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471133 2024-11-13T01:31:11,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,140 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,140 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471133) roll requested 2024-11-13T01:31:11,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471140 2024-11-13T01:31:11,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471133 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471140 2024-11-13T01:31:11,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471140) roll requested 2024-11-13T01:31:11,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471149 2024-11-13T01:31:11,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,155 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,155 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,155 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471140 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471149 2024-11-13T01:31:11,156 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,156 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,156 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,156 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471149) roll requested 2024-11-13T01:31:11,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471157 2024-11-13T01:31:11,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471157 2024-11-13T01:31:11,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471157) roll requested 2024-11-13T01:31:11,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471164 2024-11-13T01:31:11,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471164 2024-11-13T01:31:11,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471164) roll requested 2024-11-13T01:31:11,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471174 2024-11-13T01:31:11,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471174 2024-11-13T01:31:11,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471174) roll requested 2024-11-13T01:31:11,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471180 2024-11-13T01:31:11,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471180 2024-11-13T01:31:11,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471180) roll requested 2024-11-13T01:31:11,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471185 2024-11-13T01:31:11,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471185 2024-11-13T01:31:11,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,191 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471185) roll requested 2024-11-13T01:31:11,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471191 2024-11-13T01:31:11,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471191 2024-11-13T01:31:11,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471191) roll requested 2024-11-13T01:31:11,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471196 2024-11-13T01:31:11,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471196 2024-11-13T01:31:11,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471196) roll requested 2024-11-13T01:31:11,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471204 2024-11-13T01:31:11,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471204 2024-11-13T01:31:11,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,209 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,209 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471204) roll requested 2024-11-13T01:31:11,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471209 2024-11-13T01:31:11,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471204 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471209 2024-11-13T01:31:11,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,215 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,215 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471209) roll requested 2024-11-13T01:31:11,215 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471215 2024-11-13T01:31:11,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471215 2024-11-13T01:31:11,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471215) roll requested 2024-11-13T01:31:11,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471221 2024-11-13T01:31:11,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471215 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471221 2024-11-13T01:31:11,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471221) roll requested 2024-11-13T01:31:11,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471226 2024-11-13T01:31:11,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471226 2024-11-13T01:31:11,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471226) roll requested 2024-11-13T01:31:11,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471231 2024-11-13T01:31:11,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471231 2024-11-13T01:31:11,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471231) roll requested 2024-11-13T01:31:11,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471237 2024-11-13T01:31:11,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471237 2024-11-13T01:31:11,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,242 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,242 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471237) roll requested 2024-11-13T01:31:11,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471242 2024-11-13T01:31:11,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,248 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,248 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471242 2024-11-13T01:31:11,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,249 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,249 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,249 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471242) roll requested 2024-11-13T01:31:11,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471249 2024-11-13T01:31:11,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471249 2024-11-13T01:31:11,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471249) roll requested 2024-11-13T01:31:11,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471256 2024-11-13T01:31:11,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471249 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471256 2024-11-13T01:31:11,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471256) roll requested 2024-11-13T01:31:11,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471263 2024-11-13T01:31:11,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471263 2024-11-13T01:31:11,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471263) roll requested 2024-11-13T01:31:11,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471271 2024-11-13T01:31:11,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471271 2024-11-13T01:31:11,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471271) roll requested 2024-11-13T01:31:11,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471279 2024-11-13T01:31:11,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471279 2024-11-13T01:31:11,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471279) roll requested 2024-11-13T01:31:11,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471288 2024-11-13T01:31:11,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471288 2024-11-13T01:31:11,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471288) roll requested 2024-11-13T01:31:11,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471296 2024-11-13T01:31:11,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471296 2024-11-13T01:31:11,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471296) roll requested 2024-11-13T01:31:11,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471305 2024-11-13T01:31:11,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471305 2024-11-13T01:31:11,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471305) roll requested 2024-11-13T01:31:11,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471312 2024-11-13T01:31:11,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471312 2024-11-13T01:31:11,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471312) roll requested 2024-11-13T01:31:11,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471319 2024-11-13T01:31:11,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471319 2024-11-13T01:31:11,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471319) roll requested 2024-11-13T01:31:11,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471325 2024-11-13T01:31:11,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471325 2024-11-13T01:31:11,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,334 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,334 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471325) roll requested 2024-11-13T01:31:11,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471334 2024-11-13T01:31:11,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471334 2024-11-13T01:31:11,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471334) roll requested 2024-11-13T01:31:11,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471341 2024-11-13T01:31:11,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471341 2024-11-13T01:31:11,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471341) roll requested 2024-11-13T01:31:11,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471349 2024-11-13T01:31:11,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471349 2024-11-13T01:31:11,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471349) roll requested 2024-11-13T01:31:11,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471357 2024-11-13T01:31:11,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471357 2024-11-13T01:31:11,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,365 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471357) roll requested 2024-11-13T01:31:11,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471366 2024-11-13T01:31:11,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471366 2024-11-13T01:31:11,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471366) roll requested 2024-11-13T01:31:11,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471375 2024-11-13T01:31:11,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471375 2024-11-13T01:31:11,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471375) roll requested 2024-11-13T01:31:11,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471384 2024-11-13T01:31:11,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471384 2024-11-13T01:31:11,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,391 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471384) roll requested 2024-11-13T01:31:11,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471391 2024-11-13T01:31:11,397 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471391 2024-11-13T01:31:11,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,399 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,399 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,399 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471391) roll requested 2024-11-13T01:31:11,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471399 2024-11-13T01:31:11,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471391 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471399 2024-11-13T01:31:11,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471399) roll requested 2024-11-13T01:31:11,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471408 2024-11-13T01:31:11,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471399 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471408 2024-11-13T01:31:11,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471408) roll requested 2024-11-13T01:31:11,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471417 2024-11-13T01:31:11,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471417 2024-11-13T01:31:11,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471417) roll requested 2024-11-13T01:31:11,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471425 2024-11-13T01:31:11,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471425 2024-11-13T01:31:11,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471425) roll requested 2024-11-13T01:31:11,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471433 2024-11-13T01:31:11,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471433 2024-11-13T01:31:11,441 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471433) roll requested 2024-11-13T01:31:11,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471442 2024-11-13T01:31:11,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471442 2024-11-13T01:31:11,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,451 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471442) roll requested 2024-11-13T01:31:11,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471451 2024-11-13T01:31:11,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471451 2024-11-13T01:31:11,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471451) roll requested 2024-11-13T01:31:11,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471457 2024-11-13T01:31:11,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471451 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471457 2024-11-13T01:31:11,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471457) roll requested 2024-11-13T01:31:11,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471463 2024-11-13T01:31:11,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471463 2024-11-13T01:31:11,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471463) roll requested 2024-11-13T01:31:11,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471471 2024-11-13T01:31:11,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471471 2024-11-13T01:31:11,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,478 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,478 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471471) roll requested 2024-11-13T01:31:11,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471478 2024-11-13T01:31:11,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471471 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471478 2024-11-13T01:31:11,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471478) roll requested 2024-11-13T01:31:11,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471488 2024-11-13T01:31:11,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471488 2024-11-13T01:31:11,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471488) roll requested 2024-11-13T01:31:11,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471496 2024-11-13T01:31:11,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471496 2024-11-13T01:31:11,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471496) roll requested 2024-11-13T01:31:11,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471504 2024-11-13T01:31:11,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471504 2024-11-13T01:31:11,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471504) roll requested 2024-11-13T01:31:11,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471511 2024-11-13T01:31:11,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471511 2024-11-13T01:31:11,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471511) roll requested 2024-11-13T01:31:11,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471517 2024-11-13T01:31:11,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471517 2024-11-13T01:31:11,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471517) roll requested 2024-11-13T01:31:11,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471523 2024-11-13T01:31:11,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471523 2024-11-13T01:31:11,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471523) roll requested 2024-11-13T01:31:11,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471528 2024-11-13T01:31:11,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471528 2024-11-13T01:31:11,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,534 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,534 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,534 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471528) roll requested 2024-11-13T01:31:11,534 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471534 2024-11-13T01:31:11,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471534 2024-11-13T01:31:11,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471534) roll requested 2024-11-13T01:31:11,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471541 2024-11-13T01:31:11,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,548 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471534 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471541 2024-11-13T01:31:11,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471541) roll requested 2024-11-13T01:31:11,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471549 2024-11-13T01:31:11,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471549 2024-11-13T01:31:11,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471549) roll requested 2024-11-13T01:31:11,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471556 2024-11-13T01:31:11,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471556 2024-11-13T01:31:11,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471556) roll requested 2024-11-13T01:31:11,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471564 2024-11-13T01:31:11,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471564 2024-11-13T01:31:11,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,574 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,574 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,574 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471564) roll requested 2024-11-13T01:31:11,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471574 2024-11-13T01:31:11,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471574 2024-11-13T01:31:11,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471574) roll requested 2024-11-13T01:31:11,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471584 2024-11-13T01:31:11,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471574 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471584 2024-11-13T01:31:11,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,593 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471584) roll requested 2024-11-13T01:31:11,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471593 2024-11-13T01:31:11,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471593 2024-11-13T01:31:11,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471593) roll requested 2024-11-13T01:31:11,600 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471600 2024-11-13T01:31:11,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471600 2024-11-13T01:31:11,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471600) roll requested 2024-11-13T01:31:11,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471607 2024-11-13T01:31:11,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471600 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471607 2024-11-13T01:31:11,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,613 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471607) roll requested 2024-11-13T01:31:11,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471614 2024-11-13T01:31:11,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471614 2024-11-13T01:31:11,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471614) roll requested 2024-11-13T01:31:11,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471621 2024-11-13T01:31:11,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471621 2024-11-13T01:31:11,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471621) roll requested 2024-11-13T01:31:11,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471630 2024-11-13T01:31:11,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471630 2024-11-13T01:31:11,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471630) roll requested 2024-11-13T01:31:11,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471639 2024-11-13T01:31:11,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471639 2024-11-13T01:31:11,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471639) roll requested 2024-11-13T01:31:11,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471646 2024-11-13T01:31:11,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471646 2024-11-13T01:31:11,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471646) roll requested 2024-11-13T01:31:11,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471654 2024-11-13T01:31:11,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471654 2024-11-13T01:31:11,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471654) roll requested 2024-11-13T01:31:11,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471663 2024-11-13T01:31:11,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471663 2024-11-13T01:31:11,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471663) roll requested 2024-11-13T01:31:11,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471671 2024-11-13T01:31:11,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471671 2024-11-13T01:31:11,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471671) roll requested 2024-11-13T01:31:11,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471680 2024-11-13T01:31:11,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471680 2024-11-13T01:31:11,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471680) roll requested 2024-11-13T01:31:11,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471691 2024-11-13T01:31:11,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471691 2024-11-13T01:31:11,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471691) roll requested 2024-11-13T01:31:11,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471700 2024-11-13T01:31:11,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471700 2024-11-13T01:31:11,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471700) roll requested 2024-11-13T01:31:11,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471709 2024-11-13T01:31:11,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471709 2024-11-13T01:31:11,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471709) roll requested 2024-11-13T01:31:11,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471719 2024-11-13T01:31:11,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471719 2024-11-13T01:31:11,728 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,729 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,729 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,729 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471719) roll requested 2024-11-13T01:31:11,729 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471729 2024-11-13T01:31:11,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471729 2024-11-13T01:31:11,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471729) roll requested 2024-11-13T01:31:11,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471739 2024-11-13T01:31:11,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471729 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471739 2024-11-13T01:31:11,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471739) roll requested 2024-11-13T01:31:11,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471749 2024-11-13T01:31:11,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471749 2024-11-13T01:31:11,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471749) roll requested 2024-11-13T01:31:11,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471759 2024-11-13T01:31:11,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471759 2024-11-13T01:31:11,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471759) roll requested 2024-11-13T01:31:11,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471769 2024-11-13T01:31:11,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471769 2024-11-13T01:31:11,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471769) roll requested 2024-11-13T01:31:11,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471779 2024-11-13T01:31:11,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471779 2024-11-13T01:31:11,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471779) roll requested 2024-11-13T01:31:11,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471788 2024-11-13T01:31:11,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471788 2024-11-13T01:31:11,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471788) roll requested 2024-11-13T01:31:11,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471799 2024-11-13T01:31:11,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471799 2024-11-13T01:31:11,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471799) roll requested 2024-11-13T01:31:11,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471807 2024-11-13T01:31:11,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471807 2024-11-13T01:31:11,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471807) roll requested 2024-11-13T01:31:11,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471814 2024-11-13T01:31:11,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471814 2024-11-13T01:31:11,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471814) roll requested 2024-11-13T01:31:11,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471824 2024-11-13T01:31:11,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471824 2024-11-13T01:31:11,832 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471824) roll requested 2024-11-13T01:31:11,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471833 2024-11-13T01:31:11,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471833 2024-11-13T01:31:11,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471833) roll requested 2024-11-13T01:31:11,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471840 2024-11-13T01:31:11,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471840 2024-11-13T01:31:11,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471840) roll requested 2024-11-13T01:31:11,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471848 2024-11-13T01:31:11,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471848 2024-11-13T01:31:11,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,854 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471848) roll requested 2024-11-13T01:31:11,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471855 2024-11-13T01:31:11,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471855 2024-11-13T01:31:11,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471855) roll requested 2024-11-13T01:31:11,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471861 2024-11-13T01:31:11,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471861 2024-11-13T01:31:11,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471861) roll requested 2024-11-13T01:31:11,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471870 2024-11-13T01:31:11,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471870 2024-11-13T01:31:11,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,879 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471870) roll requested 2024-11-13T01:31:11,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471879 2024-11-13T01:31:11,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471879 2024-11-13T01:31:11,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471879) roll requested 2024-11-13T01:31:11,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471887 2024-11-13T01:31:11,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,893 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471887 2024-11-13T01:31:11,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,894 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,894 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,894 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,894 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,894 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471887) roll requested 2024-11-13T01:31:11,894 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471894 2024-11-13T01:31:11,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471894 2024-11-13T01:31:11,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471894) roll requested 2024-11-13T01:31:11,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471902 2024-11-13T01:31:11,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471894 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471902 2024-11-13T01:31:11,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,910 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471902) roll requested 2024-11-13T01:31:11,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471911 2024-11-13T01:31:11,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471911 2024-11-13T01:31:11,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471911) roll requested 2024-11-13T01:31:11,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471921 2024-11-13T01:31:11,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471921 2024-11-13T01:31:11,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,930 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471921) roll requested 2024-11-13T01:31:11,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471931 2024-11-13T01:31:11,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471931 2024-11-13T01:31:11,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,940 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471931) roll requested 2024-11-13T01:31:11,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471941 2024-11-13T01:31:11,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471941 2024-11-13T01:31:11,950 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471941) roll requested 2024-11-13T01:31:11,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471951 2024-11-13T01:31:11,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471951 2024-11-13T01:31:11,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471951) roll requested 2024-11-13T01:31:11,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471962 2024-11-13T01:31:11,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471962 2024-11-13T01:31:11,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,969 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,969 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,969 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471962) roll requested 2024-11-13T01:31:11,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471969 2024-11-13T01:31:11,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471969 2024-11-13T01:31:11,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471969) roll requested 2024-11-13T01:31:11,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471978 2024-11-13T01:31:11,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471969 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471978 2024-11-13T01:31:11,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471978) roll requested 2024-11-13T01:31:11,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471988 2024-11-13T01:31:11,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:11,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471988 2024-11-13T01:31:11,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:11,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:11,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:11,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:11,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471988) roll requested 2024-11-13T01:31:11,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461471997 2024-11-13T01:31:12,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471997 2024-11-13T01:31:12,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461471997) roll requested 2024-11-13T01:31:12,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472003 2024-11-13T01:31:12,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461471997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472003 2024-11-13T01:31:12,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472003) roll requested 2024-11-13T01:31:12,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472011 2024-11-13T01:31:12,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472011 2024-11-13T01:31:12,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472011) roll requested 2024-11-13T01:31:12,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472019 2024-11-13T01:31:12,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472019 2024-11-13T01:31:12,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472019) roll requested 2024-11-13T01:31:12,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472028 2024-11-13T01:31:12,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472028 2024-11-13T01:31:12,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472028) roll requested 2024-11-13T01:31:12,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472037 2024-11-13T01:31:12,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472037 2024-11-13T01:31:12,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472037) roll requested 2024-11-13T01:31:12,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472044 2024-11-13T01:31:12,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472044 2024-11-13T01:31:12,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472044) roll requested 2024-11-13T01:31:12,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472052 2024-11-13T01:31:12,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472052 2024-11-13T01:31:12,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472052) roll requested 2024-11-13T01:31:12,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472062 2024-11-13T01:31:12,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472062 2024-11-13T01:31:12,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472062) roll requested 2024-11-13T01:31:12,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472071 2024-11-13T01:31:12,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472071 2024-11-13T01:31:12,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472071) roll requested 2024-11-13T01:31:12,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472079 2024-11-13T01:31:12,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472071 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472079 2024-11-13T01:31:12,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,089 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472079) roll requested 2024-11-13T01:31:12,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472089 2024-11-13T01:31:12,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472089 2024-11-13T01:31:12,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472089) roll requested 2024-11-13T01:31:12,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472098 2024-11-13T01:31:12,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472098 2024-11-13T01:31:12,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472098) roll requested 2024-11-13T01:31:12,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472108 2024-11-13T01:31:12,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472108 2024-11-13T01:31:12,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472108) roll requested 2024-11-13T01:31:12,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472116 2024-11-13T01:31:12,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472116 2024-11-13T01:31:12,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472116) roll requested 2024-11-13T01:31:12,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472121 2024-11-13T01:31:12,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472121 2024-11-13T01:31:12,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472121) roll requested 2024-11-13T01:31:12,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472128 2024-11-13T01:31:12,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472128 2024-11-13T01:31:12,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472128) roll requested 2024-11-13T01:31:12,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472135 2024-11-13T01:31:12,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472135 2024-11-13T01:31:12,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472135) roll requested 2024-11-13T01:31:12,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472143 2024-11-13T01:31:12,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472143 2024-11-13T01:31:12,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472143) roll requested 2024-11-13T01:31:12,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472151 2024-11-13T01:31:12,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472151 2024-11-13T01:31:12,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472151) roll requested 2024-11-13T01:31:12,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472159 2024-11-13T01:31:12,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472159 2024-11-13T01:31:12,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472159) roll requested 2024-11-13T01:31:12,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472168 2024-11-13T01:31:12,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472168 2024-11-13T01:31:12,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472168) roll requested 2024-11-13T01:31:12,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472178 2024-11-13T01:31:12,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472178 2024-11-13T01:31:12,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472178) roll requested 2024-11-13T01:31:12,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472188 2024-11-13T01:31:12,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472188 2024-11-13T01:31:12,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472188) roll requested 2024-11-13T01:31:12,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472198 2024-11-13T01:31:12,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472198 2024-11-13T01:31:12,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472198) roll requested 2024-11-13T01:31:12,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472209 2024-11-13T01:31:12,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472209 2024-11-13T01:31:12,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472209) roll requested 2024-11-13T01:31:12,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472216 2024-11-13T01:31:12,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472209 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472216 2024-11-13T01:31:12,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472216) roll requested 2024-11-13T01:31:12,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472223 2024-11-13T01:31:12,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472223 2024-11-13T01:31:12,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472223) roll requested 2024-11-13T01:31:12,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472232 2024-11-13T01:31:12,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472232 2024-11-13T01:31:12,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472232) roll requested 2024-11-13T01:31:12,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472240 2024-11-13T01:31:12,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472240 2024-11-13T01:31:12,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,246 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472240) roll requested 2024-11-13T01:31:12,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472247 2024-11-13T01:31:12,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,253 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,253 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,253 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,253 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,253 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472247 2024-11-13T01:31:12,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,254 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,254 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,254 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472247) roll requested 2024-11-13T01:31:12,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472254 2024-11-13T01:31:12,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472254 2024-11-13T01:31:12,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472254) roll requested 2024-11-13T01:31:12,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472263 2024-11-13T01:31:12,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472254 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472263 2024-11-13T01:31:12,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,270 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,270 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,270 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472263) roll requested 2024-11-13T01:31:12,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472271 2024-11-13T01:31:12,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472271 2024-11-13T01:31:12,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472271) roll requested 2024-11-13T01:31:12,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472278 2024-11-13T01:31:12,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472278 2024-11-13T01:31:12,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472278) roll requested 2024-11-13T01:31:12,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472284 2024-11-13T01:31:12,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472284 2024-11-13T01:31:12,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472284) roll requested 2024-11-13T01:31:12,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472292 2024-11-13T01:31:12,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472292 2024-11-13T01:31:12,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472292) roll requested 2024-11-13T01:31:12,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472301 2024-11-13T01:31:12,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472301 2024-11-13T01:31:12,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,310 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,310 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472301) roll requested 2024-11-13T01:31:12,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472310 2024-11-13T01:31:12,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472310 2024-11-13T01:31:12,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472310) roll requested 2024-11-13T01:31:12,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472320 2024-11-13T01:31:12,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472310 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472320 2024-11-13T01:31:12,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472320) roll requested 2024-11-13T01:31:12,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472331 2024-11-13T01:31:12,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472331 2024-11-13T01:31:12,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472331) roll requested 2024-11-13T01:31:12,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472339 2024-11-13T01:31:12,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472339 2024-11-13T01:31:12,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472339) roll requested 2024-11-13T01:31:12,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472348 2024-11-13T01:31:12,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472348 2024-11-13T01:31:12,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,357 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,357 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,357 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472348) roll requested 2024-11-13T01:31:12,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472358 2024-11-13T01:31:12,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472358 2024-11-13T01:31:12,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,366 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472358) roll requested 2024-11-13T01:31:12,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472367 2024-11-13T01:31:12,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472358 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472367 2024-11-13T01:31:12,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472367) roll requested 2024-11-13T01:31:12,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472376 2024-11-13T01:31:12,384 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472376 2024-11-13T01:31:12,385 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,386 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,386 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472376) roll requested 2024-11-13T01:31:12,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472386 2024-11-13T01:31:12,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472386 2024-11-13T01:31:12,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472386) roll requested 2024-11-13T01:31:12,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472397 2024-11-13T01:31:12,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472386 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472397 2024-11-13T01:31:12,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472397) roll requested 2024-11-13T01:31:12,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472403 2024-11-13T01:31:12,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472403 2024-11-13T01:31:12,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,409 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,409 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,409 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472403) roll requested 2024-11-13T01:31:12,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472409 2024-11-13T01:31:12,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472409 2024-11-13T01:31:12,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472409) roll requested 2024-11-13T01:31:12,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472417 2024-11-13T01:31:12,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472409 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472417 2024-11-13T01:31:12,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472417) roll requested 2024-11-13T01:31:12,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472424 2024-11-13T01:31:12,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472424 2024-11-13T01:31:12,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,430 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472424) roll requested 2024-11-13T01:31:12,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472431 2024-11-13T01:31:12,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472431 2024-11-13T01:31:12,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472431) roll requested 2024-11-13T01:31:12,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472440 2024-11-13T01:31:12,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472440 2024-11-13T01:31:12,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472440) roll requested 2024-11-13T01:31:12,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472450 2024-11-13T01:31:12,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472440 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472450 2024-11-13T01:31:12,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472450) roll requested 2024-11-13T01:31:12,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472458 2024-11-13T01:31:12,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472458 2024-11-13T01:31:12,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472458) roll requested 2024-11-13T01:31:12,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472466 2024-11-13T01:31:12,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,473 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,473 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,473 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,473 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472466 2024-11-13T01:31:12,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,474 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,474 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,474 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,474 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,474 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472466) roll requested 2024-11-13T01:31:12,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472474 2024-11-13T01:31:12,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472474 2024-11-13T01:31:12,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472474) roll requested 2024-11-13T01:31:12,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472484 2024-11-13T01:31:12,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472474 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472484 2024-11-13T01:31:12,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472484) roll requested 2024-11-13T01:31:12,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472493 2024-11-13T01:31:12,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472493 2024-11-13T01:31:12,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,501 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472493) roll requested 2024-11-13T01:31:12,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472502 2024-11-13T01:31:12,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472502 2024-11-13T01:31:12,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472502) roll requested 2024-11-13T01:31:12,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472511 2024-11-13T01:31:12,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472511 2024-11-13T01:31:12,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,519 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,519 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472511) roll requested 2024-11-13T01:31:12,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472519 2024-11-13T01:31:12,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472519 2024-11-13T01:31:12,527 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,527 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472519) roll requested 2024-11-13T01:31:12,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472528 2024-11-13T01:31:12,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472519 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472528 2024-11-13T01:31:12,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,535 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472528) roll requested 2024-11-13T01:31:12,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472536 2024-11-13T01:31:12,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472536 2024-11-13T01:31:12,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472536) roll requested 2024-11-13T01:31:12,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472544 2024-11-13T01:31:12,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472544 2024-11-13T01:31:12,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472544) roll requested 2024-11-13T01:31:12,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472552 2024-11-13T01:31:12,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472552 2024-11-13T01:31:12,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472552) roll requested 2024-11-13T01:31:12,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472560 2024-11-13T01:31:12,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472560 2024-11-13T01:31:12,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472560) roll requested 2024-11-13T01:31:12,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472569 2024-11-13T01:31:12,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472569 2024-11-13T01:31:12,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,578 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,578 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,578 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472569) roll requested 2024-11-13T01:31:12,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472578 2024-11-13T01:31:12,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472578 2024-11-13T01:31:12,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472578) roll requested 2024-11-13T01:31:12,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472586 2024-11-13T01:31:12,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472586 2024-11-13T01:31:12,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472586) roll requested 2024-11-13T01:31:12,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472594 2024-11-13T01:31:12,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472594 2024-11-13T01:31:12,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472594) roll requested 2024-11-13T01:31:12,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472603 2024-11-13T01:31:12,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472603 2024-11-13T01:31:12,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472603) roll requested 2024-11-13T01:31:12,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472609 2024-11-13T01:31:12,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472609 2024-11-13T01:31:12,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,615 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,615 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,615 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472609) roll requested 2024-11-13T01:31:12,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472616 2024-11-13T01:31:12,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472616 2024-11-13T01:31:12,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472616) roll requested 2024-11-13T01:31:12,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472624 2024-11-13T01:31:12,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472624 2024-11-13T01:31:12,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472624) roll requested 2024-11-13T01:31:12,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472632 2024-11-13T01:31:12,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472632 2024-11-13T01:31:12,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,642 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472632) roll requested 2024-11-13T01:31:12,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472642 2024-11-13T01:31:12,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472632 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472642 2024-11-13T01:31:12,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472642) roll requested 2024-11-13T01:31:12,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472651 2024-11-13T01:31:12,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472651 2024-11-13T01:31:12,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472651) roll requested 2024-11-13T01:31:12,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472658 2024-11-13T01:31:12,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472658 2024-11-13T01:31:12,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472658) roll requested 2024-11-13T01:31:12,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472664 2024-11-13T01:31:12,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472664 2024-11-13T01:31:12,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472664) roll requested 2024-11-13T01:31:12,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472671 2024-11-13T01:31:12,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472671 2024-11-13T01:31:12,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,679 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,679 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472671) roll requested 2024-11-13T01:31:12,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472679 2024-11-13T01:31:12,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472679 2024-11-13T01:31:12,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472679) roll requested 2024-11-13T01:31:12,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472685 2024-11-13T01:31:12,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472679 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472685 2024-11-13T01:31:12,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,692 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,692 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472685) roll requested 2024-11-13T01:31:12,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472692 2024-11-13T01:31:12,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472692 2024-11-13T01:31:12,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472692) roll requested 2024-11-13T01:31:12,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472700 2024-11-13T01:31:12,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472692 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472700 2024-11-13T01:31:12,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472700) roll requested 2024-11-13T01:31:12,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472707 2024-11-13T01:31:12,714 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,714 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,714 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472707 2024-11-13T01:31:12,715 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472707) roll requested 2024-11-13T01:31:12,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472716 2024-11-13T01:31:12,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472716 2024-11-13T01:31:12,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472716) roll requested 2024-11-13T01:31:12,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472724 2024-11-13T01:31:12,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472724 2024-11-13T01:31:12,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472724) roll requested 2024-11-13T01:31:12,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472732 2024-11-13T01:31:12,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472732 2024-11-13T01:31:12,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472732) roll requested 2024-11-13T01:31:12,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472740 2024-11-13T01:31:12,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472740 2024-11-13T01:31:12,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472740) roll requested 2024-11-13T01:31:12,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472748 2024-11-13T01:31:12,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472748 2024-11-13T01:31:12,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472748) roll requested 2024-11-13T01:31:12,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472757 2024-11-13T01:31:12,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472757 2024-11-13T01:31:12,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472757) roll requested 2024-11-13T01:31:12,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472764 2024-11-13T01:31:12,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472764 2024-11-13T01:31:12,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,772 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472764) roll requested 2024-11-13T01:31:12,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472773 2024-11-13T01:31:12,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472773 2024-11-13T01:31:12,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472773) roll requested 2024-11-13T01:31:12,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472781 2024-11-13T01:31:12,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,786 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472781 2024-11-13T01:31:12,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,787 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,787 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,787 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472781) roll requested 2024-11-13T01:31:12,787 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472787 2024-11-13T01:31:12,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472787 2024-11-13T01:31:12,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472787) roll requested 2024-11-13T01:31:12,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472793 2024-11-13T01:31:12,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472787 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472793 2024-11-13T01:31:12,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472793) roll requested 2024-11-13T01:31:12,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472802 2024-11-13T01:31:12,807 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,807 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472802 2024-11-13T01:31:12,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472802) roll requested 2024-11-13T01:31:12,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472808 2024-11-13T01:31:12,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,814 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472808 2024-11-13T01:31:12,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472808) roll requested 2024-11-13T01:31:12,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472815 2024-11-13T01:31:12,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472815 2024-11-13T01:31:12,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472815) roll requested 2024-11-13T01:31:12,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472823 2024-11-13T01:31:12,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472823 2024-11-13T01:31:12,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,830 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472823) roll requested 2024-11-13T01:31:12,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472831 2024-11-13T01:31:12,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472831 2024-11-13T01:31:12,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472831) roll requested 2024-11-13T01:31:12,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472840 2024-11-13T01:31:12,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472840 2024-11-13T01:31:12,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,848 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,848 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,848 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472840) roll requested 2024-11-13T01:31:12,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472848 2024-11-13T01:31:12,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472848 2024-11-13T01:31:12,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,855 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,855 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,855 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472848) roll requested 2024-11-13T01:31:12,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472855 2024-11-13T01:31:12,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472848 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472855 2024-11-13T01:31:12,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472855) roll requested 2024-11-13T01:31:12,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472860 2024-11-13T01:31:12,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472855 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472860 2024-11-13T01:31:12,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472860) roll requested 2024-11-13T01:31:12,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472866 2024-11-13T01:31:12,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472866 2024-11-13T01:31:12,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472866) roll requested 2024-11-13T01:31:12,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472873 2024-11-13T01:31:12,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472873 2024-11-13T01:31:12,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472873) roll requested 2024-11-13T01:31:12,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472882 2024-11-13T01:31:12,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472882 2024-11-13T01:31:12,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472882) roll requested 2024-11-13T01:31:12,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472890 2024-11-13T01:31:12,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472890 2024-11-13T01:31:12,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472890) roll requested 2024-11-13T01:31:12,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472902 2024-11-13T01:31:12,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472902 2024-11-13T01:31:12,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,910 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472902) roll requested 2024-11-13T01:31:12,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472911 2024-11-13T01:31:12,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472911 2024-11-13T01:31:12,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472911) roll requested 2024-11-13T01:31:12,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472917 2024-11-13T01:31:12,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472917 2024-11-13T01:31:12,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472917) roll requested 2024-11-13T01:31:12,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472925 2024-11-13T01:31:12,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,932 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,932 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472925 2024-11-13T01:31:12,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,933 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,933 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,933 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472925) roll requested 2024-11-13T01:31:12,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472934 2024-11-13T01:31:12,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472934 2024-11-13T01:31:12,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472934) roll requested 2024-11-13T01:31:12,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472943 2024-11-13T01:31:12,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472943 2024-11-13T01:31:12,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472943) roll requested 2024-11-13T01:31:12,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472952 2024-11-13T01:31:12,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472952 2024-11-13T01:31:12,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,959 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472952) roll requested 2024-11-13T01:31:12,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472960 2024-11-13T01:31:12,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472960 2024-11-13T01:31:12,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472960) roll requested 2024-11-13T01:31:12,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472968 2024-11-13T01:31:12,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,974 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472968 2024-11-13T01:31:12,975 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,975 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,975 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472968) roll requested 2024-11-13T01:31:12,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472976 2024-11-13T01:31:12,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472976 2024-11-13T01:31:12,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472976) roll requested 2024-11-13T01:31:12,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472984 2024-11-13T01:31:12,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472984 2024-11-13T01:31:12,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472984) roll requested 2024-11-13T01:31:12,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472991 2024-11-13T01:31:12,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:12,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472991 2024-11-13T01:31:12,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:12,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:12,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:12,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:12,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472991) roll requested 2024-11-13T01:31:12,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461472999 2024-11-13T01:31:13,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472999 2024-11-13T01:31:13,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,006 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461472999) roll requested 2024-11-13T01:31:13,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473007 2024-11-13T01:31:13,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461472999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473007 2024-11-13T01:31:13,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,015 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473007) roll requested 2024-11-13T01:31:13,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473015 2024-11-13T01:31:13,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473015 2024-11-13T01:31:13,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473015) roll requested 2024-11-13T01:31:13,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473024 2024-11-13T01:31:13,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473024 2024-11-13T01:31:13,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473024) roll requested 2024-11-13T01:31:13,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473034 2024-11-13T01:31:13,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473034 2024-11-13T01:31:13,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,043 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473034) roll requested 2024-11-13T01:31:13,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473044 2024-11-13T01:31:13,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473044 2024-11-13T01:31:13,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473044) roll requested 2024-11-13T01:31:13,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473051 2024-11-13T01:31:13,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473051 2024-11-13T01:31:13,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473051) roll requested 2024-11-13T01:31:13,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473059 2024-11-13T01:31:13,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473059 2024-11-13T01:31:13,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473059) roll requested 2024-11-13T01:31:13,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473067 2024-11-13T01:31:13,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473067 2024-11-13T01:31:13,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473067) roll requested 2024-11-13T01:31:13,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473073 2024-11-13T01:31:13,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473073 2024-11-13T01:31:13,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473073) roll requested 2024-11-13T01:31:13,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473081 2024-11-13T01:31:13,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473081 2024-11-13T01:31:13,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473081) roll requested 2024-11-13T01:31:13,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473089 2024-11-13T01:31:13,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473089 2024-11-13T01:31:13,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473089) roll requested 2024-11-13T01:31:13,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473098 2024-11-13T01:31:13,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473098 2024-11-13T01:31:13,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473098) roll requested 2024-11-13T01:31:13,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473108 2024-11-13T01:31:13,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473108 2024-11-13T01:31:13,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473108) roll requested 2024-11-13T01:31:13,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473119 2024-11-13T01:31:13,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473119 2024-11-13T01:31:13,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473119) roll requested 2024-11-13T01:31:13,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473128 2024-11-13T01:31:13,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473128 2024-11-13T01:31:13,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,138 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473128) roll requested 2024-11-13T01:31:13,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473138 2024-11-13T01:31:13,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473138 2024-11-13T01:31:13,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,147 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473138) roll requested 2024-11-13T01:31:13,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473147 2024-11-13T01:31:13,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473147 2024-11-13T01:31:13,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473147) roll requested 2024-11-13T01:31:13,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473155 2024-11-13T01:31:13,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473155 2024-11-13T01:31:13,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473155) roll requested 2024-11-13T01:31:13,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473166 2024-11-13T01:31:13,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473166 2024-11-13T01:31:13,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473166) roll requested 2024-11-13T01:31:13,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473177 2024-11-13T01:31:13,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473177 2024-11-13T01:31:13,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473177) roll requested 2024-11-13T01:31:13,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473184 2024-11-13T01:31:13,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473184 2024-11-13T01:31:13,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473184) roll requested 2024-11-13T01:31:13,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473189 2024-11-13T01:31:13,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473189 2024-11-13T01:31:13,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473189) roll requested 2024-11-13T01:31:13,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473195 2024-11-13T01:31:13,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473195 2024-11-13T01:31:13,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473195) roll requested 2024-11-13T01:31:13,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473203 2024-11-13T01:31:13,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473203 2024-11-13T01:31:13,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473203) roll requested 2024-11-13T01:31:13,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473211 2024-11-13T01:31:13,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473211 2024-11-13T01:31:13,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473211) roll requested 2024-11-13T01:31:13,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473218 2024-11-13T01:31:13,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473218 2024-11-13T01:31:13,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473218) roll requested 2024-11-13T01:31:13,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473228 2024-11-13T01:31:13,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473228 2024-11-13T01:31:13,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473228) roll requested 2024-11-13T01:31:13,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473238 2024-11-13T01:31:13,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473238 2024-11-13T01:31:13,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473238) roll requested 2024-11-13T01:31:13,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473245 2024-11-13T01:31:13,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473245 2024-11-13T01:31:13,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473245) roll requested 2024-11-13T01:31:13,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473252 2024-11-13T01:31:13,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473252 2024-11-13T01:31:13,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,260 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473252) roll requested 2024-11-13T01:31:13,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473260 2024-11-13T01:31:13,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473260 2024-11-13T01:31:13,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473260) roll requested 2024-11-13T01:31:13,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473269 2024-11-13T01:31:13,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473260 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473269 2024-11-13T01:31:13,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,278 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,278 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,278 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473269) roll requested 2024-11-13T01:31:13,278 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473278 2024-11-13T01:31:13,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473278 2024-11-13T01:31:13,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,287 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,287 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473278) roll requested 2024-11-13T01:31:13,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473287 2024-11-13T01:31:13,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473278 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473287 2024-11-13T01:31:13,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473287) roll requested 2024-11-13T01:31:13,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473297 2024-11-13T01:31:13,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473287 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473297 2024-11-13T01:31:13,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473297) roll requested 2024-11-13T01:31:13,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473307 2024-11-13T01:31:13,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473307 2024-11-13T01:31:13,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473307) roll requested 2024-11-13T01:31:13,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473317 2024-11-13T01:31:13,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473317 2024-11-13T01:31:13,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,327 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,327 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,327 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473317) roll requested 2024-11-13T01:31:13,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473327 2024-11-13T01:31:13,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473327 2024-11-13T01:31:13,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473327) roll requested 2024-11-13T01:31:13,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473336 2024-11-13T01:31:13,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473327 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473336 2024-11-13T01:31:13,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473336) roll requested 2024-11-13T01:31:13,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473346 2024-11-13T01:31:13,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473346 2024-11-13T01:31:13,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,355 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473346) roll requested 2024-11-13T01:31:13,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473355 2024-11-13T01:31:13,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473355 2024-11-13T01:31:13,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473355) roll requested 2024-11-13T01:31:13,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473364 2024-11-13T01:31:13,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473364 2024-11-13T01:31:13,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473364) roll requested 2024-11-13T01:31:13,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473373 2024-11-13T01:31:13,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473373 2024-11-13T01:31:13,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,380 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,380 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473373) roll requested 2024-11-13T01:31:13,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473380 2024-11-13T01:31:13,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473380 2024-11-13T01:31:13,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,387 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473380) roll requested 2024-11-13T01:31:13,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473387 2024-11-13T01:31:13,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473380 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473387 2024-11-13T01:31:13,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473387) roll requested 2024-11-13T01:31:13,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473396 2024-11-13T01:31:13,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473396 2024-11-13T01:31:13,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473396) roll requested 2024-11-13T01:31:13,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473404 2024-11-13T01:31:13,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473404 2024-11-13T01:31:13,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473404) roll requested 2024-11-13T01:31:13,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473411 2024-11-13T01:31:13,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473411 2024-11-13T01:31:13,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,419 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,419 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,419 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473411) roll requested 2024-11-13T01:31:13,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473419 2024-11-13T01:31:13,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473419 2024-11-13T01:31:13,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473419) roll requested 2024-11-13T01:31:13,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473428 2024-11-13T01:31:13,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473419 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473428 2024-11-13T01:31:13,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473428) roll requested 2024-11-13T01:31:13,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473435 2024-11-13T01:31:13,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473435 2024-11-13T01:31:13,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473435) roll requested 2024-11-13T01:31:13,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473444 2024-11-13T01:31:13,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473444 2024-11-13T01:31:13,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,453 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473444) roll requested 2024-11-13T01:31:13,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473454 2024-11-13T01:31:13,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473454 2024-11-13T01:31:13,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473454) roll requested 2024-11-13T01:31:13,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473465 2024-11-13T01:31:13,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473465 2024-11-13T01:31:13,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473465) roll requested 2024-11-13T01:31:13,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473476 2024-11-13T01:31:13,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473476 2024-11-13T01:31:13,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473476) roll requested 2024-11-13T01:31:13,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473483 2024-11-13T01:31:13,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473483 2024-11-13T01:31:13,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473483) roll requested 2024-11-13T01:31:13,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473491 2024-11-13T01:31:13,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473491 2024-11-13T01:31:13,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,499 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,499 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,499 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473491) roll requested 2024-11-13T01:31:13,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473499 2024-11-13T01:31:13,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473499 2024-11-13T01:31:13,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473499) roll requested 2024-11-13T01:31:13,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473507 2024-11-13T01:31:13,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,514 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,514 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,514 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473507 2024-11-13T01:31:13,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,515 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473507) roll requested 2024-11-13T01:31:13,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473515 2024-11-13T01:31:13,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473515 2024-11-13T01:31:13,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473515) roll requested 2024-11-13T01:31:13,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473523 2024-11-13T01:31:13,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473515 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473523 2024-11-13T01:31:13,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473523) roll requested 2024-11-13T01:31:13,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473531 2024-11-13T01:31:13,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473531 2024-11-13T01:31:13,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,540 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473531) roll requested 2024-11-13T01:31:13,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473540 2024-11-13T01:31:13,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473540 2024-11-13T01:31:13,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473540) roll requested 2024-11-13T01:31:13,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473550 2024-11-13T01:31:13,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473550 2024-11-13T01:31:13,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,560 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473550) roll requested 2024-11-13T01:31:13,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473560 2024-11-13T01:31:13,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473560 2024-11-13T01:31:13,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473560) roll requested 2024-11-13T01:31:13,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473570 2024-11-13T01:31:13,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473570 2024-11-13T01:31:13,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,578 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,578 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473570) roll requested 2024-11-13T01:31:13,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473579 2024-11-13T01:31:13,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473579 2024-11-13T01:31:13,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473579) roll requested 2024-11-13T01:31:13,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473586 2024-11-13T01:31:13,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473586 2024-11-13T01:31:13,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473586) roll requested 2024-11-13T01:31:13,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473594 2024-11-13T01:31:13,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473594 2024-11-13T01:31:13,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473594) roll requested 2024-11-13T01:31:13,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473602 2024-11-13T01:31:13,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473602 2024-11-13T01:31:13,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473602) roll requested 2024-11-13T01:31:13,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473609 2024-11-13T01:31:13,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473609 2024-11-13T01:31:13,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,616 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473609) roll requested 2024-11-13T01:31:13,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473616 2024-11-13T01:31:13,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473616 2024-11-13T01:31:13,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473616) roll requested 2024-11-13T01:31:13,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473625 2024-11-13T01:31:13,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473616 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473625 2024-11-13T01:31:13,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473625) roll requested 2024-11-13T01:31:13,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473635 2024-11-13T01:31:13,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473635 2024-11-13T01:31:13,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473635) roll requested 2024-11-13T01:31:13,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473643 2024-11-13T01:31:13,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473643 2024-11-13T01:31:13,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,651 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473643) roll requested 2024-11-13T01:31:13,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473651 2024-11-13T01:31:13,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473651 2024-11-13T01:31:13,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,659 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473651) roll requested 2024-11-13T01:31:13,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473659 2024-11-13T01:31:13,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473659 2024-11-13T01:31:13,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473659) roll requested 2024-11-13T01:31:13,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473667 2024-11-13T01:31:13,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473667 2024-11-13T01:31:13,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473667) roll requested 2024-11-13T01:31:13,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473677 2024-11-13T01:31:13,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473677 2024-11-13T01:31:13,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473677) roll requested 2024-11-13T01:31:13,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473686 2024-11-13T01:31:13,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473686 2024-11-13T01:31:13,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473686) roll requested 2024-11-13T01:31:13,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473696 2024-11-13T01:31:13,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473696 2024-11-13T01:31:13,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473696) roll requested 2024-11-13T01:31:13,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473706 2024-11-13T01:31:13,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473706 2024-11-13T01:31:13,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,713 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,713 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473706) roll requested 2024-11-13T01:31:13,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473713 2024-11-13T01:31:13,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473713 2024-11-13T01:31:13,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,719 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473713) roll requested 2024-11-13T01:31:13,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473719 2024-11-13T01:31:13,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473713 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473719 2024-11-13T01:31:13,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,723 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473719) roll requested 2024-11-13T01:31:13,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473724 2024-11-13T01:31:13,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473719 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473724 2024-11-13T01:31:13,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473724) roll requested 2024-11-13T01:31:13,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473730 2024-11-13T01:31:13,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473730 2024-11-13T01:31:13,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473730) roll requested 2024-11-13T01:31:13,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473737 2024-11-13T01:31:13,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473737 2024-11-13T01:31:13,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473737) roll requested 2024-11-13T01:31:13,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473744 2024-11-13T01:31:13,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473744 2024-11-13T01:31:13,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,754 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473744) roll requested 2024-11-13T01:31:13,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473754 2024-11-13T01:31:13,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473754 2024-11-13T01:31:13,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,762 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,762 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,762 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,762 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473754) roll requested 2024-11-13T01:31:13,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473762 2024-11-13T01:31:13,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473762 2024-11-13T01:31:13,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473762) roll requested 2024-11-13T01:31:13,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473770 2024-11-13T01:31:13,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473762 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473770 2024-11-13T01:31:13,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473770) roll requested 2024-11-13T01:31:13,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473779 2024-11-13T01:31:13,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473779 2024-11-13T01:31:13,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473779) roll requested 2024-11-13T01:31:13,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473789 2024-11-13T01:31:13,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473789 2024-11-13T01:31:13,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473789) roll requested 2024-11-13T01:31:13,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473799 2024-11-13T01:31:13,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473799 2024-11-13T01:31:13,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473799) roll requested 2024-11-13T01:31:13,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473804 2024-11-13T01:31:13,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473804 2024-11-13T01:31:13,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473804) roll requested 2024-11-13T01:31:13,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473811 2024-11-13T01:31:13,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473811 2024-11-13T01:31:13,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473811) roll requested 2024-11-13T01:31:13,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473818 2024-11-13T01:31:13,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473818 2024-11-13T01:31:13,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473818) roll requested 2024-11-13T01:31:13,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473826 2024-11-13T01:31:13,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473826 2024-11-13T01:31:13,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473826) roll requested 2024-11-13T01:31:13,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473836 2024-11-13T01:31:13,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473836 2024-11-13T01:31:13,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473836) roll requested 2024-11-13T01:31:13,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473847 2024-11-13T01:31:13,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473847 2024-11-13T01:31:13,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473847) roll requested 2024-11-13T01:31:13,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473857 2024-11-13T01:31:13,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473857 2024-11-13T01:31:13,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473857) roll requested 2024-11-13T01:31:13,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473865 2024-11-13T01:31:13,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473865 2024-11-13T01:31:13,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473865) roll requested 2024-11-13T01:31:13,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473872 2024-11-13T01:31:13,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473872 2024-11-13T01:31:13,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473872) roll requested 2024-11-13T01:31:13,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473878 2024-11-13T01:31:13,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473878 2024-11-13T01:31:13,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473878) roll requested 2024-11-13T01:31:13,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473884 2024-11-13T01:31:13,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473884 2024-11-13T01:31:13,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473884) roll requested 2024-11-13T01:31:13,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473890 2024-11-13T01:31:13,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473890 2024-11-13T01:31:13,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473890) roll requested 2024-11-13T01:31:13,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473896 2024-11-13T01:31:13,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,902 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,902 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,902 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473896 2024-11-13T01:31:13,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,903 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,903 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,903 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473896) roll requested 2024-11-13T01:31:13,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473903 2024-11-13T01:31:13,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473903 2024-11-13T01:31:13,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473903) roll requested 2024-11-13T01:31:13,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473912 2024-11-13T01:31:13,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473903 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473912 2024-11-13T01:31:13,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473912) roll requested 2024-11-13T01:31:13,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473920 2024-11-13T01:31:13,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473920 2024-11-13T01:31:13,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,929 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,929 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473920) roll requested 2024-11-13T01:31:13,929 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473929 2024-11-13T01:31:13,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473929 2024-11-13T01:31:13,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473929) roll requested 2024-11-13T01:31:13,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473936 2024-11-13T01:31:13,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473929 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473936 2024-11-13T01:31:13,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473936) roll requested 2024-11-13T01:31:13,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473942 2024-11-13T01:31:13,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473936 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473942 2024-11-13T01:31:13,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,950 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,950 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,950 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,950 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,950 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473942) roll requested 2024-11-13T01:31:13,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473950 2024-11-13T01:31:13,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473950 2024-11-13T01:31:13,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473950) roll requested 2024-11-13T01:31:13,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473958 2024-11-13T01:31:13,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473958 2024-11-13T01:31:13,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473958) roll requested 2024-11-13T01:31:13,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473967 2024-11-13T01:31:13,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,974 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,974 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473967 2024-11-13T01:31:13,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,976 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,976 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473967) roll requested 2024-11-13T01:31:13,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473976 2024-11-13T01:31:13,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473976 2024-11-13T01:31:13,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,986 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473976) roll requested 2024-11-13T01:31:13,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473987 2024-11-13T01:31:13,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:13,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473976 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473987 2024-11-13T01:31:13,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:13,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:13,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:13,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:13,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473987) roll requested 2024-11-13T01:31:13,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461473997 2024-11-13T01:31:14,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473997 2024-11-13T01:31:14,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461473997) roll requested 2024-11-13T01:31:14,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474005 2024-11-13T01:31:14,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461473997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474005 2024-11-13T01:31:14,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474005) roll requested 2024-11-13T01:31:14,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474012 2024-11-13T01:31:14,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474012 2024-11-13T01:31:14,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474012) roll requested 2024-11-13T01:31:14,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474019 2024-11-13T01:31:14,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474019 2024-11-13T01:31:14,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474019) roll requested 2024-11-13T01:31:14,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474025 2024-11-13T01:31:14,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474025 2024-11-13T01:31:14,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474025) roll requested 2024-11-13T01:31:14,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474033 2024-11-13T01:31:14,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474033 2024-11-13T01:31:14,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474033) roll requested 2024-11-13T01:31:14,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474042 2024-11-13T01:31:14,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474042 2024-11-13T01:31:14,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474042) roll requested 2024-11-13T01:31:14,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474051 2024-11-13T01:31:14,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474051 2024-11-13T01:31:14,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474051) roll requested 2024-11-13T01:31:14,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474058 2024-11-13T01:31:14,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474058 2024-11-13T01:31:14,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474058) roll requested 2024-11-13T01:31:14,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474066 2024-11-13T01:31:14,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474058 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474066 2024-11-13T01:31:14,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474066) roll requested 2024-11-13T01:31:14,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474074 2024-11-13T01:31:14,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474074 2024-11-13T01:31:14,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474074) roll requested 2024-11-13T01:31:14,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474082 2024-11-13T01:31:14,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474074 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474082 2024-11-13T01:31:14,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474082) roll requested 2024-11-13T01:31:14,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474091 2024-11-13T01:31:14,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474091 2024-11-13T01:31:14,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474091) roll requested 2024-11-13T01:31:14,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474099 2024-11-13T01:31:14,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474099 2024-11-13T01:31:14,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474099) roll requested 2024-11-13T01:31:14,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474107 2024-11-13T01:31:14,113 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474107 2024-11-13T01:31:14,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474107) roll requested 2024-11-13T01:31:14,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474115 2024-11-13T01:31:14,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474115 2024-11-13T01:31:14,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474115) roll requested 2024-11-13T01:31:14,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474125 2024-11-13T01:31:14,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474125 2024-11-13T01:31:14,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,131 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,131 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474125) roll requested 2024-11-13T01:31:14,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474132 2024-11-13T01:31:14,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,136 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474132 2024-11-13T01:31:14,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474132) roll requested 2024-11-13T01:31:14,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474137 2024-11-13T01:31:14,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474137 2024-11-13T01:31:14,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,147 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474137) roll requested 2024-11-13T01:31:14,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474148 2024-11-13T01:31:14,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474148 2024-11-13T01:31:14,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474148) roll requested 2024-11-13T01:31:14,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474155 2024-11-13T01:31:14,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474155 2024-11-13T01:31:14,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474155) roll requested 2024-11-13T01:31:14,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474162 2024-11-13T01:31:14,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474162 2024-11-13T01:31:14,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,171 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,171 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474162) roll requested 2024-11-13T01:31:14,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474172 2024-11-13T01:31:14,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474172 2024-11-13T01:31:14,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,177 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,177 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,177 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474172) roll requested 2024-11-13T01:31:14,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474178 2024-11-13T01:31:14,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,183 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,183 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474178 2024-11-13T01:31:14,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474178) roll requested 2024-11-13T01:31:14,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474184 2024-11-13T01:31:14,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474184 2024-11-13T01:31:14,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,189 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474184) roll requested 2024-11-13T01:31:14,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474190 2024-11-13T01:31:14,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474190 2024-11-13T01:31:14,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474190) roll requested 2024-11-13T01:31:14,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474196 2024-11-13T01:31:14,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,200 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,200 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,200 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474196 2024-11-13T01:31:14,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,201 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,201 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,201 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474196) roll requested 2024-11-13T01:31:14,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474201 2024-11-13T01:31:14,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474201 2024-11-13T01:31:14,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474201) roll requested 2024-11-13T01:31:14,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474208 2024-11-13T01:31:14,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474201 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474208 2024-11-13T01:31:14,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474208) roll requested 2024-11-13T01:31:14,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474213 2024-11-13T01:31:14,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474213 2024-11-13T01:31:14,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,218 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,218 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474213) roll requested 2024-11-13T01:31:14,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474218 2024-11-13T01:31:14,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474218 2024-11-13T01:31:14,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474218) roll requested 2024-11-13T01:31:14,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474225 2024-11-13T01:31:14,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474218 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474225 2024-11-13T01:31:14,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474225) roll requested 2024-11-13T01:31:14,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474232 2024-11-13T01:31:14,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474232 2024-11-13T01:31:14,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474232) roll requested 2024-11-13T01:31:14,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474240 2024-11-13T01:31:14,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474240 2024-11-13T01:31:14,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,248 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,248 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,248 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474240) roll requested 2024-11-13T01:31:14,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474248 2024-11-13T01:31:14,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474248 2024-11-13T01:31:14,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474248) roll requested 2024-11-13T01:31:14,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474257 2024-11-13T01:31:14,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474248 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474257 2024-11-13T01:31:14,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474257) roll requested 2024-11-13T01:31:14,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474269 2024-11-13T01:31:14,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474269 2024-11-13T01:31:14,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,277 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474269) roll requested 2024-11-13T01:31:14,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474277 2024-11-13T01:31:14,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,282 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,282 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,282 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474277 2024-11-13T01:31:14,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,283 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,283 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,283 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,283 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474277) roll requested 2024-11-13T01:31:14,283 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474283 2024-11-13T01:31:14,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474283 2024-11-13T01:31:14,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474283) roll requested 2024-11-13T01:31:14,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474291 2024-11-13T01:31:14,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474283 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474291 2024-11-13T01:31:14,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474291) roll requested 2024-11-13T01:31:14,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474297 2024-11-13T01:31:14,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,301 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,301 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,301 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474297 2024-11-13T01:31:14,302 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,302 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,302 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,302 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,302 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,302 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474297) roll requested 2024-11-13T01:31:14,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474302 2024-11-13T01:31:14,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474302 2024-11-13T01:31:14,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,308 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,308 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474302) roll requested 2024-11-13T01:31:14,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474308 2024-11-13T01:31:14,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474302 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474308 2024-11-13T01:31:14,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474308) roll requested 2024-11-13T01:31:14,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474318 2024-11-13T01:31:14,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474318 2024-11-13T01:31:14,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474318) roll requested 2024-11-13T01:31:14,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474325 2024-11-13T01:31:14,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474325 2024-11-13T01:31:14,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474325) roll requested 2024-11-13T01:31:14,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474331 2024-11-13T01:31:14,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474331 2024-11-13T01:31:14,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474331) roll requested 2024-11-13T01:31:14,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474337 2024-11-13T01:31:14,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474337 2024-11-13T01:31:14,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474337) roll requested 2024-11-13T01:31:14,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474343 2024-11-13T01:31:14,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474343 2024-11-13T01:31:14,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,351 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,352 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474343) roll requested 2024-11-13T01:31:14,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474352 2024-11-13T01:31:14,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474352 2024-11-13T01:31:14,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474352) roll requested 2024-11-13T01:31:14,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474360 2024-11-13T01:31:14,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474352 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474360 2024-11-13T01:31:14,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474360) roll requested 2024-11-13T01:31:14,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474367 2024-11-13T01:31:14,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474367 2024-11-13T01:31:14,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474367) roll requested 2024-11-13T01:31:14,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474373 2024-11-13T01:31:14,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474373 2024-11-13T01:31:14,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474373) roll requested 2024-11-13T01:31:14,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474424 2024-11-13T01:31:14,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474424 2024-11-13T01:31:14,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474424) roll requested 2024-11-13T01:31:14,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474436 2024-11-13T01:31:14,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474436 2024-11-13T01:31:14,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474436) roll requested 2024-11-13T01:31:14,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474443 2024-11-13T01:31:14,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474443 2024-11-13T01:31:14,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,449 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,449 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474443) roll requested 2024-11-13T01:31:14,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474449 2024-11-13T01:31:14,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474449 2024-11-13T01:31:14,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,454 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474449) roll requested 2024-11-13T01:31:14,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474455 2024-11-13T01:31:14,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474449 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474455 2024-11-13T01:31:14,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474455) roll requested 2024-11-13T01:31:14,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474460 2024-11-13T01:31:14,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474460 2024-11-13T01:31:14,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474460) roll requested 2024-11-13T01:31:14,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474465 2024-11-13T01:31:14,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474465 2024-11-13T01:31:14,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474465) roll requested 2024-11-13T01:31:14,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474472 2024-11-13T01:31:14,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474472 2024-11-13T01:31:14,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474472) roll requested 2024-11-13T01:31:14,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474480 2024-11-13T01:31:14,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474480 2024-11-13T01:31:14,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,488 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474480) roll requested 2024-11-13T01:31:14,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474488 2024-11-13T01:31:14,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474488 2024-11-13T01:31:14,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474488) roll requested 2024-11-13T01:31:14,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474497 2024-11-13T01:31:14,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474488 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474497 2024-11-13T01:31:14,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474497) roll requested 2024-11-13T01:31:14,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474507 2024-11-13T01:31:14,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474507 2024-11-13T01:31:14,518 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,518 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,518 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,518 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,518 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,518 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474507) roll requested 2024-11-13T01:31:14,519 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474518 2024-11-13T01:31:14,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474518 2024-11-13T01:31:14,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474518) roll requested 2024-11-13T01:31:14,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474527 2024-11-13T01:31:14,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474518 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474527 2024-11-13T01:31:14,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474527) roll requested 2024-11-13T01:31:14,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474536 2024-11-13T01:31:14,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474536 2024-11-13T01:31:14,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474536) roll requested 2024-11-13T01:31:14,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474548 2024-11-13T01:31:14,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474548 2024-11-13T01:31:14,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474548) roll requested 2024-11-13T01:31:14,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474557 2024-11-13T01:31:14,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474557 2024-11-13T01:31:14,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474557) roll requested 2024-11-13T01:31:14,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474568 2024-11-13T01:31:14,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,576 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,576 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474568 2024-11-13T01:31:14,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474568) roll requested 2024-11-13T01:31:14,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474578 2024-11-13T01:31:14,584 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,584 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,584 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474578 2024-11-13T01:31:14,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,586 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474578) roll requested 2024-11-13T01:31:14,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474586 2024-11-13T01:31:14,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474578 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474586 2024-11-13T01:31:14,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474586) roll requested 2024-11-13T01:31:14,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474594 2024-11-13T01:31:14,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474594 2024-11-13T01:31:14,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474594) roll requested 2024-11-13T01:31:14,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474603 2024-11-13T01:31:14,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474603 2024-11-13T01:31:14,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474603) roll requested 2024-11-13T01:31:14,612 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474612 2024-11-13T01:31:14,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474612 2024-11-13T01:31:14,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474612) roll requested 2024-11-13T01:31:14,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474621 2024-11-13T01:31:14,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474612 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474621 2024-11-13T01:31:14,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474621) roll requested 2024-11-13T01:31:14,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474630 2024-11-13T01:31:14,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474630 2024-11-13T01:31:14,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474630) roll requested 2024-11-13T01:31:14,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474635 2024-11-13T01:31:14,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474635 2024-11-13T01:31:14,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474635) roll requested 2024-11-13T01:31:14,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474640 2024-11-13T01:31:14,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474640 2024-11-13T01:31:14,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474640) roll requested 2024-11-13T01:31:14,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474647 2024-11-13T01:31:14,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474647 2024-11-13T01:31:14,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474647) roll requested 2024-11-13T01:31:14,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474654 2024-11-13T01:31:14,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474654 2024-11-13T01:31:14,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474654) roll requested 2024-11-13T01:31:14,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474663 2024-11-13T01:31:14,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474663 2024-11-13T01:31:14,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474663) roll requested 2024-11-13T01:31:14,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474670 2024-11-13T01:31:14,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474670 2024-11-13T01:31:14,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474670) roll requested 2024-11-13T01:31:14,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474678 2024-11-13T01:31:14,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474678 2024-11-13T01:31:14,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474678) roll requested 2024-11-13T01:31:14,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474688 2024-11-13T01:31:14,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474688 2024-11-13T01:31:14,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474688) roll requested 2024-11-13T01:31:14,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474697 2024-11-13T01:31:14,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474697 2024-11-13T01:31:14,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,707 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,707 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,707 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,707 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474697) roll requested 2024-11-13T01:31:14,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474707 2024-11-13T01:31:14,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474707 2024-11-13T01:31:14,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474707) roll requested 2024-11-13T01:31:14,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474717 2024-11-13T01:31:14,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474707 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474717 2024-11-13T01:31:14,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,726 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,726 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,727 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474717) roll requested 2024-11-13T01:31:14,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474727 2024-11-13T01:31:14,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474727 2024-11-13T01:31:14,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,735 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474727) roll requested 2024-11-13T01:31:14,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474736 2024-11-13T01:31:14,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474727 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474736 2024-11-13T01:31:14,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474736) roll requested 2024-11-13T01:31:14,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474744 2024-11-13T01:31:14,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474744 2024-11-13T01:31:14,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474744) roll requested 2024-11-13T01:31:14,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474751 2024-11-13T01:31:14,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474751 2024-11-13T01:31:14,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474751) roll requested 2024-11-13T01:31:14,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474758 2024-11-13T01:31:14,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474758 2024-11-13T01:31:14,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474758) roll requested 2024-11-13T01:31:14,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474766 2024-11-13T01:31:14,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474766 2024-11-13T01:31:14,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474766) roll requested 2024-11-13T01:31:14,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474773 2024-11-13T01:31:14,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474773 2024-11-13T01:31:14,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474773) roll requested 2024-11-13T01:31:14,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474782 2024-11-13T01:31:14,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474782 2024-11-13T01:31:14,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474782) roll requested 2024-11-13T01:31:14,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474793 2024-11-13T01:31:14,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474793 2024-11-13T01:31:14,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474793) roll requested 2024-11-13T01:31:14,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474801 2024-11-13T01:31:14,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474801 2024-11-13T01:31:14,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,809 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474801) roll requested 2024-11-13T01:31:14,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474810 2024-11-13T01:31:14,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474810 2024-11-13T01:31:14,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474810) roll requested 2024-11-13T01:31:14,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474818 2024-11-13T01:31:14,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474818 2024-11-13T01:31:14,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474818) roll requested 2024-11-13T01:31:14,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474829 2024-11-13T01:31:14,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474829 2024-11-13T01:31:14,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474829) roll requested 2024-11-13T01:31:14,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474841 2024-11-13T01:31:14,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,847 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474841 2024-11-13T01:31:14,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474841) roll requested 2024-11-13T01:31:14,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474852 2024-11-13T01:31:14,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474852 2024-11-13T01:31:14,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474852) roll requested 2024-11-13T01:31:14,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474858 2024-11-13T01:31:14,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474858 2024-11-13T01:31:14,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,863 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,863 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474858) roll requested 2024-11-13T01:31:14,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474863 2024-11-13T01:31:14,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474863 2024-11-13T01:31:14,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474863) roll requested 2024-11-13T01:31:14,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474869 2024-11-13T01:31:14,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474869 2024-11-13T01:31:14,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474869) roll requested 2024-11-13T01:31:14,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474875 2024-11-13T01:31:14,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474875 2024-11-13T01:31:14,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474875) roll requested 2024-11-13T01:31:14,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474881 2024-11-13T01:31:14,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474881 2024-11-13T01:31:14,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,886 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,886 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,886 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,886 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474881) roll requested 2024-11-13T01:31:14,886 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474886 2024-11-13T01:31:14,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474886 2024-11-13T01:31:14,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474886) roll requested 2024-11-13T01:31:14,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474892 2024-11-13T01:31:14,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474886 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474892 2024-11-13T01:31:14,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474892) roll requested 2024-11-13T01:31:14,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474901 2024-11-13T01:31:14,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474901 2024-11-13T01:31:14,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474901) roll requested 2024-11-13T01:31:14,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474907 2024-11-13T01:31:14,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474907 2024-11-13T01:31:14,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,912 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,912 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474907) roll requested 2024-11-13T01:31:14,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474912 2024-11-13T01:31:14,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474912 2024-11-13T01:31:14,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474912) roll requested 2024-11-13T01:31:14,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474918 2024-11-13T01:31:14,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474912 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474918 2024-11-13T01:31:14,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474918) roll requested 2024-11-13T01:31:14,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474924 2024-11-13T01:31:14,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474924 2024-11-13T01:31:14,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474924) roll requested 2024-11-13T01:31:14,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474930 2024-11-13T01:31:14,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474930 2024-11-13T01:31:14,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474930) roll requested 2024-11-13T01:31:14,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474937 2024-11-13T01:31:14,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474937 2024-11-13T01:31:14,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474937) roll requested 2024-11-13T01:31:14,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474943 2024-11-13T01:31:14,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474943 2024-11-13T01:31:14,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474943) roll requested 2024-11-13T01:31:14,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474948 2024-11-13T01:31:14,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474948 2024-11-13T01:31:14,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,953 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,953 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474948) roll requested 2024-11-13T01:31:14,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474953 2024-11-13T01:31:14,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474948 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474953 2024-11-13T01:31:14,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474953) roll requested 2024-11-13T01:31:14,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474958 2024-11-13T01:31:14,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474953 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474958 2024-11-13T01:31:14,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,965 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,965 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474958) roll requested 2024-11-13T01:31:14,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474965 2024-11-13T01:31:14,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474965 2024-11-13T01:31:14,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474965) roll requested 2024-11-13T01:31:14,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474972 2024-11-13T01:31:14,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474965 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474972 2024-11-13T01:31:14,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,979 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474972) roll requested 2024-11-13T01:31:14,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474979 2024-11-13T01:31:14,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474979 2024-11-13T01:31:14,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474979) roll requested 2024-11-13T01:31:14,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474985 2024-11-13T01:31:14,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474985 2024-11-13T01:31:14,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474985) roll requested 2024-11-13T01:31:14,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474992 2024-11-13T01:31:14,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474992 2024-11-13T01:31:14,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:14,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:14,998 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:14,998 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:14,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474992) roll requested 2024-11-13T01:31:14,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461474998 2024-11-13T01:31:15,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474998 2024-11-13T01:31:15,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,003 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461474998) roll requested 2024-11-13T01:31:15,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475004 2024-11-13T01:31:15,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461474998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475004 2024-11-13T01:31:15,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475004) roll requested 2024-11-13T01:31:15,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475009 2024-11-13T01:31:15,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475009 2024-11-13T01:31:15,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475009) roll requested 2024-11-13T01:31:15,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475017 2024-11-13T01:31:15,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,023 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475017 2024-11-13T01:31:15,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,024 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,024 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,024 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475017) roll requested 2024-11-13T01:31:15,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475024 2024-11-13T01:31:15,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475024 2024-11-13T01:31:15,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475024) roll requested 2024-11-13T01:31:15,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475031 2024-11-13T01:31:15,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475031 2024-11-13T01:31:15,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475031) roll requested 2024-11-13T01:31:15,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475039 2024-11-13T01:31:15,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475039 2024-11-13T01:31:15,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475039) roll requested 2024-11-13T01:31:15,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475048 2024-11-13T01:31:15,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475048 2024-11-13T01:31:15,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475048) roll requested 2024-11-13T01:31:15,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475056 2024-11-13T01:31:15,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475056 2024-11-13T01:31:15,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,062 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475056) roll requested 2024-11-13T01:31:15,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475063 2024-11-13T01:31:15,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475056 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475063 2024-11-13T01:31:15,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475063) roll requested 2024-11-13T01:31:15,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475069 2024-11-13T01:31:15,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475069 2024-11-13T01:31:15,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475069) roll requested 2024-11-13T01:31:15,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475077 2024-11-13T01:31:15,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475077 2024-11-13T01:31:15,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475077) roll requested 2024-11-13T01:31:15,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475086 2024-11-13T01:31:15,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475086 2024-11-13T01:31:15,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,095 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475086) roll requested 2024-11-13T01:31:15,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475096 2024-11-13T01:31:15,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475096 2024-11-13T01:31:15,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475096) roll requested 2024-11-13T01:31:15,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475104 2024-11-13T01:31:15,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475104 2024-11-13T01:31:15,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475104) roll requested 2024-11-13T01:31:15,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475111 2024-11-13T01:31:15,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475111 2024-11-13T01:31:15,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475111) roll requested 2024-11-13T01:31:15,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475119 2024-11-13T01:31:15,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475119 2024-11-13T01:31:15,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475119) roll requested 2024-11-13T01:31:15,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475128 2024-11-13T01:31:15,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475128 2024-11-13T01:31:15,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475128) roll requested 2024-11-13T01:31:15,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475137 2024-11-13T01:31:15,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475137 2024-11-13T01:31:15,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475137) roll requested 2024-11-13T01:31:15,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475144 2024-11-13T01:31:15,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,149 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475144 2024-11-13T01:31:15,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475144) roll requested 2024-11-13T01:31:15,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475151 2024-11-13T01:31:15,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475151 2024-11-13T01:31:15,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475151) roll requested 2024-11-13T01:31:15,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475160 2024-11-13T01:31:15,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,168 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,168 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,168 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,168 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,168 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475160 2024-11-13T01:31:15,169 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,169 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475160) roll requested 2024-11-13T01:31:15,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475170 2024-11-13T01:31:15,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475170 2024-11-13T01:31:15,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,179 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,179 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475170) roll requested 2024-11-13T01:31:15,179 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475179 2024-11-13T01:31:15,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475170 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475179 2024-11-13T01:31:15,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475179) roll requested 2024-11-13T01:31:15,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475189 2024-11-13T01:31:15,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,195 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475179 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475189 2024-11-13T01:31:15,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475189) roll requested 2024-11-13T01:31:15,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475196 2024-11-13T01:31:15,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475196 2024-11-13T01:31:15,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475196) roll requested 2024-11-13T01:31:15,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475203 2024-11-13T01:31:15,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475203 2024-11-13T01:31:15,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475203) roll requested 2024-11-13T01:31:15,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475211 2024-11-13T01:31:15,217 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475211 2024-11-13T01:31:15,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,218 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,219 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475211) roll requested 2024-11-13T01:31:15,219 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475219 2024-11-13T01:31:15,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475219 2024-11-13T01:31:15,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,228 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,228 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,228 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,228 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475219) roll requested 2024-11-13T01:31:15,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475228 2024-11-13T01:31:15,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475219 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475228 2024-11-13T01:31:15,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475228) roll requested 2024-11-13T01:31:15,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475238 2024-11-13T01:31:15,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475228 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475238 2024-11-13T01:31:15,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475238) roll requested 2024-11-13T01:31:15,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475251 2024-11-13T01:31:15,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475251 2024-11-13T01:31:15,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475251) roll requested 2024-11-13T01:31:15,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475259 2024-11-13T01:31:15,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475259 2024-11-13T01:31:15,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475259) roll requested 2024-11-13T01:31:15,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475267 2024-11-13T01:31:15,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475267 2024-11-13T01:31:15,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475267) roll requested 2024-11-13T01:31:15,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475274 2024-11-13T01:31:15,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475274 2024-11-13T01:31:15,282 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,282 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,282 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475274) roll requested 2024-11-13T01:31:15,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475282 2024-11-13T01:31:15,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475282 2024-11-13T01:31:15,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475282) roll requested 2024-11-13T01:31:15,292 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475292 2024-11-13T01:31:15,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475292 2024-11-13T01:31:15,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475292) roll requested 2024-11-13T01:31:15,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475300 2024-11-13T01:31:15,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475292 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475300 2024-11-13T01:31:15,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475300) roll requested 2024-11-13T01:31:15,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475307 2024-11-13T01:31:15,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475307 2024-11-13T01:31:15,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475307) roll requested 2024-11-13T01:31:15,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475316 2024-11-13T01:31:15,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475316 2024-11-13T01:31:15,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475316) roll requested 2024-11-13T01:31:15,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475323 2024-11-13T01:31:15,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475323 2024-11-13T01:31:15,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,331 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,331 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475323) roll requested 2024-11-13T01:31:15,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475331 2024-11-13T01:31:15,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475331 2024-11-13T01:31:15,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475331) roll requested 2024-11-13T01:31:15,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475336 2024-11-13T01:31:15,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475336 2024-11-13T01:31:15,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475336) roll requested 2024-11-13T01:31:15,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475342 2024-11-13T01:31:15,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475342 2024-11-13T01:31:15,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,349 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475342) roll requested 2024-11-13T01:31:15,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475349 2024-11-13T01:31:15,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475349 2024-11-13T01:31:15,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475349) roll requested 2024-11-13T01:31:15,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475361 2024-11-13T01:31:15,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475361 2024-11-13T01:31:15,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,369 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,369 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475361) roll requested 2024-11-13T01:31:15,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475369 2024-11-13T01:31:15,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475369 2024-11-13T01:31:15,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,376 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,377 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,377 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475369) roll requested 2024-11-13T01:31:15,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475377 2024-11-13T01:31:15,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475369 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475377 2024-11-13T01:31:15,382 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,382 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475377) roll requested 2024-11-13T01:31:15,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475383 2024-11-13T01:31:15,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475377 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475383 2024-11-13T01:31:15,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475383) roll requested 2024-11-13T01:31:15,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475389 2024-11-13T01:31:15,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475389 2024-11-13T01:31:15,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475389) roll requested 2024-11-13T01:31:15,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475395 2024-11-13T01:31:15,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475395 2024-11-13T01:31:15,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475395) roll requested 2024-11-13T01:31:15,402 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475402 2024-11-13T01:31:15,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475402 2024-11-13T01:31:15,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475402) roll requested 2024-11-13T01:31:15,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475408 2024-11-13T01:31:15,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475408 2024-11-13T01:31:15,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475408) roll requested 2024-11-13T01:31:15,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475417 2024-11-13T01:31:15,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475417 2024-11-13T01:31:15,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475417) roll requested 2024-11-13T01:31:15,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475423 2024-11-13T01:31:15,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475423 2024-11-13T01:31:15,431 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475423) roll requested 2024-11-13T01:31:15,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475431 2024-11-13T01:31:15,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475431 2024-11-13T01:31:15,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,437 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475431) roll requested 2024-11-13T01:31:15,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475437 2024-11-13T01:31:15,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475437 2024-11-13T01:31:15,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475437) roll requested 2024-11-13T01:31:15,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475442 2024-11-13T01:31:15,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475437 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475442 2024-11-13T01:31:15,447 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,447 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475442) roll requested 2024-11-13T01:31:15,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475448 2024-11-13T01:31:15,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475448 2024-11-13T01:31:15,454 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475448) roll requested 2024-11-13T01:31:15,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475455 2024-11-13T01:31:15,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475455 2024-11-13T01:31:15,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,463 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,463 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475455) roll requested 2024-11-13T01:31:15,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475463 2024-11-13T01:31:15,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475455 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475463 2024-11-13T01:31:15,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475463) roll requested 2024-11-13T01:31:15,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475473 2024-11-13T01:31:15,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475463 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475473 2024-11-13T01:31:15,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475473) roll requested 2024-11-13T01:31:15,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475483 2024-11-13T01:31:15,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475483 2024-11-13T01:31:15,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475483) roll requested 2024-11-13T01:31:15,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475491 2024-11-13T01:31:15,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,497 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475491 2024-11-13T01:31:15,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475491) roll requested 2024-11-13T01:31:15,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475498 2024-11-13T01:31:15,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475498 2024-11-13T01:31:15,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475498) roll requested 2024-11-13T01:31:15,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475507 2024-11-13T01:31:15,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475507 2024-11-13T01:31:15,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475507) roll requested 2024-11-13T01:31:15,518 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475517 2024-11-13T01:31:15,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475517 2024-11-13T01:31:15,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475517) roll requested 2024-11-13T01:31:15,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475527 2024-11-13T01:31:15,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475527 2024-11-13T01:31:15,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,536 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,536 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475527) roll requested 2024-11-13T01:31:15,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475536 2024-11-13T01:31:15,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475536 2024-11-13T01:31:15,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475536) roll requested 2024-11-13T01:31:15,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475545 2024-11-13T01:31:15,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475536 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475545 2024-11-13T01:31:15,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475545) roll requested 2024-11-13T01:31:15,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475552 2024-11-13T01:31:15,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475552 2024-11-13T01:31:15,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475552) roll requested 2024-11-13T01:31:15,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475560 2024-11-13T01:31:15,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475560 2024-11-13T01:31:15,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475560) roll requested 2024-11-13T01:31:15,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475568 2024-11-13T01:31:15,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475560 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475568 2024-11-13T01:31:15,577 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,577 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,577 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,577 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475568) roll requested 2024-11-13T01:31:15,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475577 2024-11-13T01:31:15,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475577 2024-11-13T01:31:15,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475577) roll requested 2024-11-13T01:31:15,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475585 2024-11-13T01:31:15,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475585 2024-11-13T01:31:15,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,593 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,593 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475585) roll requested 2024-11-13T01:31:15,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475594 2024-11-13T01:31:15,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475594 2024-11-13T01:31:15,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,603 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,603 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475594) roll requested 2024-11-13T01:31:15,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475603 2024-11-13T01:31:15,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475603 2024-11-13T01:31:15,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475603) roll requested 2024-11-13T01:31:15,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475611 2024-11-13T01:31:15,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475611 2024-11-13T01:31:15,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475611) roll requested 2024-11-13T01:31:15,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475621 2024-11-13T01:31:15,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475621 2024-11-13T01:31:15,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,629 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,629 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,629 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475621) roll requested 2024-11-13T01:31:15,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475629 2024-11-13T01:31:15,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475629 2024-11-13T01:31:15,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,638 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,638 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,638 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475629) roll requested 2024-11-13T01:31:15,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475639 2024-11-13T01:31:15,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475629 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475639 2024-11-13T01:31:15,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475639) roll requested 2024-11-13T01:31:15,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475649 2024-11-13T01:31:15,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475649 2024-11-13T01:31:15,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,659 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475649) roll requested 2024-11-13T01:31:15,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475659 2024-11-13T01:31:15,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475659 2024-11-13T01:31:15,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475659) roll requested 2024-11-13T01:31:15,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475664 2024-11-13T01:31:15,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475664 2024-11-13T01:31:15,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,669 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475664) roll requested 2024-11-13T01:31:15,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475669 2024-11-13T01:31:15,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475669 2024-11-13T01:31:15,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,676 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,677 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475669) roll requested 2024-11-13T01:31:15,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475677 2024-11-13T01:31:15,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475669 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475677 2024-11-13T01:31:15,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475677) roll requested 2024-11-13T01:31:15,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475685 2024-11-13T01:31:15,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,691 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,691 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,691 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,691 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,691 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475677 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475685 2024-11-13T01:31:15,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475685) roll requested 2024-11-13T01:31:15,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475693 2024-11-13T01:31:15,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475693 2024-11-13T01:31:15,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475693) roll requested 2024-11-13T01:31:15,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475702 2024-11-13T01:31:15,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475702 2024-11-13T01:31:15,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475702) roll requested 2024-11-13T01:31:15,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475709 2024-11-13T01:31:15,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475709 2024-11-13T01:31:15,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475709) roll requested 2024-11-13T01:31:15,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475717 2024-11-13T01:31:15,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475717 2024-11-13T01:31:15,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475717) roll requested 2024-11-13T01:31:15,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475725 2024-11-13T01:31:15,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475725 2024-11-13T01:31:15,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475725) roll requested 2024-11-13T01:31:15,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475730 2024-11-13T01:31:15,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475730 2024-11-13T01:31:15,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475730) roll requested 2024-11-13T01:31:15,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475736 2024-11-13T01:31:15,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475736 2024-11-13T01:31:15,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475736) roll requested 2024-11-13T01:31:15,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475745 2024-11-13T01:31:15,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475745 2024-11-13T01:31:15,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475745) roll requested 2024-11-13T01:31:15,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475752 2024-11-13T01:31:15,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475752 2024-11-13T01:31:15,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475752) roll requested 2024-11-13T01:31:15,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475758 2024-11-13T01:31:15,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475758 2024-11-13T01:31:15,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475758) roll requested 2024-11-13T01:31:15,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475766 2024-11-13T01:31:15,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475766 2024-11-13T01:31:15,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475766) roll requested 2024-11-13T01:31:15,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475775 2024-11-13T01:31:15,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475775 2024-11-13T01:31:15,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475775) roll requested 2024-11-13T01:31:15,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475784 2024-11-13T01:31:15,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475784 2024-11-13T01:31:15,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475784) roll requested 2024-11-13T01:31:15,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475793 2024-11-13T01:31:15,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475793 2024-11-13T01:31:15,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,801 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,801 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475793) roll requested 2024-11-13T01:31:15,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475801 2024-11-13T01:31:15,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475801 2024-11-13T01:31:15,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475801) roll requested 2024-11-13T01:31:15,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475808 2024-11-13T01:31:15,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475808 2024-11-13T01:31:15,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,816 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,816 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475808) roll requested 2024-11-13T01:31:15,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475816 2024-11-13T01:31:15,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475816 2024-11-13T01:31:15,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,824 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475816) roll requested 2024-11-13T01:31:15,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475824 2024-11-13T01:31:15,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,831 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475824 2024-11-13T01:31:15,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,832 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475824) roll requested 2024-11-13T01:31:15,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475832 2024-11-13T01:31:15,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475832 2024-11-13T01:31:15,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475832) roll requested 2024-11-13T01:31:15,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475841 2024-11-13T01:31:15,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475841 2024-11-13T01:31:15,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475841) roll requested 2024-11-13T01:31:15,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475847 2024-11-13T01:31:15,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475847 2024-11-13T01:31:15,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475847) roll requested 2024-11-13T01:31:15,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475853 2024-11-13T01:31:15,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475853 2024-11-13T01:31:15,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475853) roll requested 2024-11-13T01:31:15,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475861 2024-11-13T01:31:15,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475861 2024-11-13T01:31:15,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475861) roll requested 2024-11-13T01:31:15,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475870 2024-11-13T01:31:15,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475870 2024-11-13T01:31:15,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,879 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,879 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475870) roll requested 2024-11-13T01:31:15,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475879 2024-11-13T01:31:15,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475879 2024-11-13T01:31:15,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475879) roll requested 2024-11-13T01:31:15,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475889 2024-11-13T01:31:15,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475879 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475889 2024-11-13T01:31:15,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475889) roll requested 2024-11-13T01:31:15,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475896 2024-11-13T01:31:15,901 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,901 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,901 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,901 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475896 2024-11-13T01:31:15,902 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,902 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,902 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475896) roll requested 2024-11-13T01:31:15,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475902 2024-11-13T01:31:15,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475902 2024-11-13T01:31:15,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475902) roll requested 2024-11-13T01:31:15,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475908 2024-11-13T01:31:15,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475908 2024-11-13T01:31:15,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,915 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,915 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475908) roll requested 2024-11-13T01:31:15,915 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475915 2024-11-13T01:31:15,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475915 2024-11-13T01:31:15,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475915) roll requested 2024-11-13T01:31:15,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475921 2024-11-13T01:31:15,928 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475915 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475921 2024-11-13T01:31:15,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,929 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,930 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,930 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475921) roll requested 2024-11-13T01:31:15,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475930 2024-11-13T01:31:15,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475930 2024-11-13T01:31:15,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,938 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475930) roll requested 2024-11-13T01:31:15,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475938 2024-11-13T01:31:15,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475930 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475938 2024-11-13T01:31:15,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475938) roll requested 2024-11-13T01:31:15,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475946 2024-11-13T01:31:15,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475946 2024-11-13T01:31:15,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475946) roll requested 2024-11-13T01:31:15,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475954 2024-11-13T01:31:15,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,961 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475954 2024-11-13T01:31:15,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,962 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,962 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,962 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475954) roll requested 2024-11-13T01:31:15,962 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475962 2024-11-13T01:31:15,969 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,969 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,969 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,969 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475962 2024-11-13T01:31:15,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475962) roll requested 2024-11-13T01:31:15,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475971 2024-11-13T01:31:15,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475962 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475971 2024-11-13T01:31:15,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,981 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475971) roll requested 2024-11-13T01:31:15,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475981 2024-11-13T01:31:15,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475981 2024-11-13T01:31:15,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:15,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:15,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:15,991 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:15,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475981) roll requested 2024-11-13T01:31:15,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461475991 2024-11-13T01:31:15,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:15,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475981 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475991 2024-11-13T01:31:16,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461475991) roll requested 2024-11-13T01:31:16,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476001 2024-11-13T01:31:16,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461475991 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476001 2024-11-13T01:31:16,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476001) roll requested 2024-11-13T01:31:16,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476010 2024-11-13T01:31:16,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,018 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476010 2024-11-13T01:31:16,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476010) roll requested 2024-11-13T01:31:16,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476020 2024-11-13T01:31:16,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,028 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476020 2024-11-13T01:31:16,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476020) roll requested 2024-11-13T01:31:16,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476030 2024-11-13T01:31:16,037 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476030 2024-11-13T01:31:16,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,039 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476030) roll requested 2024-11-13T01:31:16,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476039 2024-11-13T01:31:16,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476039 2024-11-13T01:31:16,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476039) roll requested 2024-11-13T01:31:16,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476047 2024-11-13T01:31:16,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476039 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476047 2024-11-13T01:31:16,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476047) roll requested 2024-11-13T01:31:16,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476057 2024-11-13T01:31:16,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476057 2024-11-13T01:31:16,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476057) roll requested 2024-11-13T01:31:16,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476065 2024-11-13T01:31:16,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476065 2024-11-13T01:31:16,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476065) roll requested 2024-11-13T01:31:16,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476073 2024-11-13T01:31:16,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476065 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476073 2024-11-13T01:31:16,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476073) roll requested 2024-11-13T01:31:16,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476082 2024-11-13T01:31:16,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,090 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476082 2024-11-13T01:31:16,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,091 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,091 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,091 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,091 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476082) roll requested 2024-11-13T01:31:16,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476091 2024-11-13T01:31:16,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476091 2024-11-13T01:31:16,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,099 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,099 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476091) roll requested 2024-11-13T01:31:16,099 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476099 2024-11-13T01:31:16,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476099 2024-11-13T01:31:16,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476099) roll requested 2024-11-13T01:31:16,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476107 2024-11-13T01:31:16,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476099 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476107 2024-11-13T01:31:16,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476107) roll requested 2024-11-13T01:31:16,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476116 2024-11-13T01:31:16,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476116 2024-11-13T01:31:16,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476116) roll requested 2024-11-13T01:31:16,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476125 2024-11-13T01:31:16,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476125 2024-11-13T01:31:16,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476125) roll requested 2024-11-13T01:31:16,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476135 2024-11-13T01:31:16,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476135 2024-11-13T01:31:16,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476135) roll requested 2024-11-13T01:31:16,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476146 2024-11-13T01:31:16,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476146 2024-11-13T01:31:16,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476146) roll requested 2024-11-13T01:31:16,156 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476155 2024-11-13T01:31:16,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476155 2024-11-13T01:31:16,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,164 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476155) roll requested 2024-11-13T01:31:16,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476164 2024-11-13T01:31:16,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476164 2024-11-13T01:31:16,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476164) roll requested 2024-11-13T01:31:16,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476173 2024-11-13T01:31:16,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476164 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476173 2024-11-13T01:31:16,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476173) roll requested 2024-11-13T01:31:16,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476184 2024-11-13T01:31:16,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476184 2024-11-13T01:31:16,194 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,194 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476184) roll requested 2024-11-13T01:31:16,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476195 2024-11-13T01:31:16,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476195 2024-11-13T01:31:16,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,206 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476195) roll requested 2024-11-13T01:31:16,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476206 2024-11-13T01:31:16,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476206 2024-11-13T01:31:16,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476206) roll requested 2024-11-13T01:31:16,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476216 2024-11-13T01:31:16,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476216 2024-11-13T01:31:16,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476216) roll requested 2024-11-13T01:31:16,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476226 2024-11-13T01:31:16,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476226 2024-11-13T01:31:16,234 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,234 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,234 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,234 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,234 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476226) roll requested 2024-11-13T01:31:16,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476235 2024-11-13T01:31:16,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476235 2024-11-13T01:31:16,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,243 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476235) roll requested 2024-11-13T01:31:16,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476244 2024-11-13T01:31:16,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,251 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476244 2024-11-13T01:31:16,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476244) roll requested 2024-11-13T01:31:16,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476253 2024-11-13T01:31:16,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476253 2024-11-13T01:31:16,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,261 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,261 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476253) roll requested 2024-11-13T01:31:16,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476261 2024-11-13T01:31:16,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476261 2024-11-13T01:31:16,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476261) roll requested 2024-11-13T01:31:16,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476268 2024-11-13T01:31:16,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476268 2024-11-13T01:31:16,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,277 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,277 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476268) roll requested 2024-11-13T01:31:16,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476277 2024-11-13T01:31:16,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,282 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476277 2024-11-13T01:31:16,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,283 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476277) roll requested 2024-11-13T01:31:16,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476284 2024-11-13T01:31:16,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476284 2024-11-13T01:31:16,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476284) roll requested 2024-11-13T01:31:16,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476293 2024-11-13T01:31:16,301 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,301 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,301 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476293 2024-11-13T01:31:16,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476293) roll requested 2024-11-13T01:31:16,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476304 2024-11-13T01:31:16,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476304 2024-11-13T01:31:16,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,315 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476304) roll requested 2024-11-13T01:31:16,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476315 2024-11-13T01:31:16,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476315 2024-11-13T01:31:16,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476315) roll requested 2024-11-13T01:31:16,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476323 2024-11-13T01:31:16,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476323 2024-11-13T01:31:16,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476323) roll requested 2024-11-13T01:31:16,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476329 2024-11-13T01:31:16,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,336 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,336 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476329 2024-11-13T01:31:16,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,338 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476329) roll requested 2024-11-13T01:31:16,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476338 2024-11-13T01:31:16,346 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,346 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,346 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,346 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,346 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476338 2024-11-13T01:31:16,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,347 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476338) roll requested 2024-11-13T01:31:16,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476348 2024-11-13T01:31:16,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476338 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476348 2024-11-13T01:31:16,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476348) roll requested 2024-11-13T01:31:16,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476356 2024-11-13T01:31:16,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476348 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476356 2024-11-13T01:31:16,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,365 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476356) roll requested 2024-11-13T01:31:16,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476365 2024-11-13T01:31:16,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476365 2024-11-13T01:31:16,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476365) roll requested 2024-11-13T01:31:16,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476372 2024-11-13T01:31:16,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476372 2024-11-13T01:31:16,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,380 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476372) roll requested 2024-11-13T01:31:16,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476381 2024-11-13T01:31:16,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476381 2024-11-13T01:31:16,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476381) roll requested 2024-11-13T01:31:16,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476389 2024-11-13T01:31:16,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476389 2024-11-13T01:31:16,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476389) roll requested 2024-11-13T01:31:16,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476396 2024-11-13T01:31:16,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476396 2024-11-13T01:31:16,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476396) roll requested 2024-11-13T01:31:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476404 2024-11-13T01:31:16,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476404 2024-11-13T01:31:16,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476404) roll requested 2024-11-13T01:31:16,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476412 2024-11-13T01:31:16,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476412 2024-11-13T01:31:16,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476412) roll requested 2024-11-13T01:31:16,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476423 2024-11-13T01:31:16,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476423 2024-11-13T01:31:16,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,432 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,432 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476423) roll requested 2024-11-13T01:31:16,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476433 2024-11-13T01:31:16,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476433 2024-11-13T01:31:16,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476433) roll requested 2024-11-13T01:31:16,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476442 2024-11-13T01:31:16,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476442 2024-11-13T01:31:16,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476442) roll requested 2024-11-13T01:31:16,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476450 2024-11-13T01:31:16,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476450 2024-11-13T01:31:16,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,459 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476450) roll requested 2024-11-13T01:31:16,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476459 2024-11-13T01:31:16,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476459 2024-11-13T01:31:16,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476459) roll requested 2024-11-13T01:31:16,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476468 2024-11-13T01:31:16,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476468 2024-11-13T01:31:16,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476468) roll requested 2024-11-13T01:31:16,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476475 2024-11-13T01:31:16,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476475 2024-11-13T01:31:16,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476475) roll requested 2024-11-13T01:31:16,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476484 2024-11-13T01:31:16,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476484 2024-11-13T01:31:16,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476484) roll requested 2024-11-13T01:31:16,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476493 2024-11-13T01:31:16,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476484 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476493 2024-11-13T01:31:16,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476493) roll requested 2024-11-13T01:31:16,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476504 2024-11-13T01:31:16,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476504 2024-11-13T01:31:16,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476504) roll requested 2024-11-13T01:31:16,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476514 2024-11-13T01:31:16,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476514 2024-11-13T01:31:16,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476514) roll requested 2024-11-13T01:31:16,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476522 2024-11-13T01:31:16,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476522 2024-11-13T01:31:16,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,529 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,529 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476522) roll requested 2024-11-13T01:31:16,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476529 2024-11-13T01:31:16,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476529 2024-11-13T01:31:16,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476529) roll requested 2024-11-13T01:31:16,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476538 2024-11-13T01:31:16,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476529 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476538 2024-11-13T01:31:16,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476538) roll requested 2024-11-13T01:31:16,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476547 2024-11-13T01:31:16,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476547 2024-11-13T01:31:16,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476547) roll requested 2024-11-13T01:31:16,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476556 2024-11-13T01:31:16,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476547 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476556 2024-11-13T01:31:16,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476556) roll requested 2024-11-13T01:31:16,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476563 2024-11-13T01:31:16,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476563 2024-11-13T01:31:16,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476563) roll requested 2024-11-13T01:31:16,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476571 2024-11-13T01:31:16,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476571 2024-11-13T01:31:16,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476571) roll requested 2024-11-13T01:31:16,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476580 2024-11-13T01:31:16,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476580 2024-11-13T01:31:16,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476580) roll requested 2024-11-13T01:31:16,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476589 2024-11-13T01:31:16,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476589 2024-11-13T01:31:16,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476589) roll requested 2024-11-13T01:31:16,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476595 2024-11-13T01:31:16,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476595 2024-11-13T01:31:16,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,602 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476595) roll requested 2024-11-13T01:31:16,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476603 2024-11-13T01:31:16,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476603 2024-11-13T01:31:16,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476603) roll requested 2024-11-13T01:31:16,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476610 2024-11-13T01:31:16,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476603 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476610 2024-11-13T01:31:16,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476610) roll requested 2024-11-13T01:31:16,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476620 2024-11-13T01:31:16,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476620 2024-11-13T01:31:16,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476620) roll requested 2024-11-13T01:31:16,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476628 2024-11-13T01:31:16,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476628 2024-11-13T01:31:16,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476628) roll requested 2024-11-13T01:31:16,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476636 2024-11-13T01:31:16,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476636 2024-11-13T01:31:16,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,646 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,646 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476636) roll requested 2024-11-13T01:31:16,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476646 2024-11-13T01:31:16,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476636 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476646 2024-11-13T01:31:16,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476646) roll requested 2024-11-13T01:31:16,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476654 2024-11-13T01:31:16,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476646 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476654 2024-11-13T01:31:16,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,662 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,662 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476654) roll requested 2024-11-13T01:31:16,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476663 2024-11-13T01:31:16,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476654 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476663 2024-11-13T01:31:16,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476663) roll requested 2024-11-13T01:31:16,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476672 2024-11-13T01:31:16,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,679 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,679 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476672 2024-11-13T01:31:16,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476672) roll requested 2024-11-13T01:31:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476681 2024-11-13T01:31:16,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476681 2024-11-13T01:31:16,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476681) roll requested 2024-11-13T01:31:16,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476688 2024-11-13T01:31:16,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476688 2024-11-13T01:31:16,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,695 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476688) roll requested 2024-11-13T01:31:16,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476696 2024-11-13T01:31:16,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476696 2024-11-13T01:31:16,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476696) roll requested 2024-11-13T01:31:16,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476704 2024-11-13T01:31:16,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476704 2024-11-13T01:31:16,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476704) roll requested 2024-11-13T01:31:16,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476717 2024-11-13T01:31:16,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476717 2024-11-13T01:31:16,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476717) roll requested 2024-11-13T01:31:16,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476728 2024-11-13T01:31:16,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476728 2024-11-13T01:31:16,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476728) roll requested 2024-11-13T01:31:16,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476734 2024-11-13T01:31:16,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476734 2024-11-13T01:31:16,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476734) roll requested 2024-11-13T01:31:16,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476741 2024-11-13T01:31:16,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476741 2024-11-13T01:31:16,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476741) roll requested 2024-11-13T01:31:16,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476749 2024-11-13T01:31:16,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476749 2024-11-13T01:31:16,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476749) roll requested 2024-11-13T01:31:16,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476757 2024-11-13T01:31:16,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476757 2024-11-13T01:31:16,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476757) roll requested 2024-11-13T01:31:16,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476767 2024-11-13T01:31:16,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476767 2024-11-13T01:31:16,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476767) roll requested 2024-11-13T01:31:16,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476774 2024-11-13T01:31:16,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476774 2024-11-13T01:31:16,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476774) roll requested 2024-11-13T01:31:16,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476780 2024-11-13T01:31:16,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476780 2024-11-13T01:31:16,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,785 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476780) roll requested 2024-11-13T01:31:16,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476786 2024-11-13T01:31:16,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476780 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476786 2024-11-13T01:31:16,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476786) roll requested 2024-11-13T01:31:16,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476796 2024-11-13T01:31:16,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476796 2024-11-13T01:31:16,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476796) roll requested 2024-11-13T01:31:16,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476805 2024-11-13T01:31:16,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476805 2024-11-13T01:31:16,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476805) roll requested 2024-11-13T01:31:16,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476814 2024-11-13T01:31:16,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476814 2024-11-13T01:31:16,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476814) roll requested 2024-11-13T01:31:16,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476820 2024-11-13T01:31:16,825 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,825 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,825 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476814 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476820 2024-11-13T01:31:16,826 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,826 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,826 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,826 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,826 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476820) roll requested 2024-11-13T01:31:16,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476826 2024-11-13T01:31:16,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476826 2024-11-13T01:31:16,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476826) roll requested 2024-11-13T01:31:16,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476834 2024-11-13T01:31:16,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476826 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476834 2024-11-13T01:31:16,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476834) roll requested 2024-11-13T01:31:16,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476841 2024-11-13T01:31:16,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476841 2024-11-13T01:31:16,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476841) roll requested 2024-11-13T01:31:16,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476850 2024-11-13T01:31:16,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476850 2024-11-13T01:31:16,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476850) roll requested 2024-11-13T01:31:16,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476861 2024-11-13T01:31:16,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476861 2024-11-13T01:31:16,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476861) roll requested 2024-11-13T01:31:16,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476870 2024-11-13T01:31:16,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476870 2024-11-13T01:31:16,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476870) roll requested 2024-11-13T01:31:16,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476878 2024-11-13T01:31:16,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476878 2024-11-13T01:31:16,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,885 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,885 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476878) roll requested 2024-11-13T01:31:16,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476885 2024-11-13T01:31:16,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476885 2024-11-13T01:31:16,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476885) roll requested 2024-11-13T01:31:16,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476893 2024-11-13T01:31:16,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476893 2024-11-13T01:31:16,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,902 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,902 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,902 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476893) roll requested 2024-11-13T01:31:16,902 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476902 2024-11-13T01:31:16,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476902 2024-11-13T01:31:16,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476902) roll requested 2024-11-13T01:31:16,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476911 2024-11-13T01:31:16,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476902 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476911 2024-11-13T01:31:16,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,918 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,918 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476911) roll requested 2024-11-13T01:31:16,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476919 2024-11-13T01:31:16,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476919 2024-11-13T01:31:16,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476919) roll requested 2024-11-13T01:31:16,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476925 2024-11-13T01:31:16,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476919 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476925 2024-11-13T01:31:16,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,932 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,932 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,932 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476925) roll requested 2024-11-13T01:31:16,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476932 2024-11-13T01:31:16,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476932 2024-11-13T01:31:16,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476932) roll requested 2024-11-13T01:31:16,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476943 2024-11-13T01:31:16,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476943 2024-11-13T01:31:16,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476943) roll requested 2024-11-13T01:31:16,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476949 2024-11-13T01:31:16,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476949 2024-11-13T01:31:16,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476949) roll requested 2024-11-13T01:31:16,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476954 2024-11-13T01:31:16,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476954 2024-11-13T01:31:16,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476954) roll requested 2024-11-13T01:31:16,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476960 2024-11-13T01:31:16,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476960 2024-11-13T01:31:16,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,968 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,968 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,968 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476960) roll requested 2024-11-13T01:31:16,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476968 2024-11-13T01:31:16,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476968 2024-11-13T01:31:16,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476968) roll requested 2024-11-13T01:31:16,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476977 2024-11-13T01:31:16,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476968 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476977 2024-11-13T01:31:16,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476977) roll requested 2024-11-13T01:31:16,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476988 2024-11-13T01:31:16,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:16,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476988 2024-11-13T01:31:16,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:16,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:16,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:16,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:16,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476988) roll requested 2024-11-13T01:31:16,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461476999 2024-11-13T01:31:17,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476999 2024-11-13T01:31:17,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,008 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,008 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461476999) roll requested 2024-11-13T01:31:17,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477009 2024-11-13T01:31:17,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461476999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477009 2024-11-13T01:31:17,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477009) roll requested 2024-11-13T01:31:17,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477018 2024-11-13T01:31:17,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477018 2024-11-13T01:31:17,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477018) roll requested 2024-11-13T01:31:17,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477029 2024-11-13T01:31:17,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477029 2024-11-13T01:31:17,038 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,038 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,038 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477029) roll requested 2024-11-13T01:31:17,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477038 2024-11-13T01:31:17,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477038 2024-11-13T01:31:17,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477038) roll requested 2024-11-13T01:31:17,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477047 2024-11-13T01:31:17,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477047 2024-11-13T01:31:17,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477047) roll requested 2024-11-13T01:31:17,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477055 2024-11-13T01:31:17,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477055 2024-11-13T01:31:17,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477055) roll requested 2024-11-13T01:31:17,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477061 2024-11-13T01:31:17,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477061 2024-11-13T01:31:17,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477061) roll requested 2024-11-13T01:31:17,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477067 2024-11-13T01:31:17,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477067 2024-11-13T01:31:17,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477067) roll requested 2024-11-13T01:31:17,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477073 2024-11-13T01:31:17,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477073 2024-11-13T01:31:17,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,080 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,080 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477073) roll requested 2024-11-13T01:31:17,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477081 2024-11-13T01:31:17,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477081 2024-11-13T01:31:17,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477081) roll requested 2024-11-13T01:31:17,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477086 2024-11-13T01:31:17,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477086 2024-11-13T01:31:17,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,092 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,092 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477086) roll requested 2024-11-13T01:31:17,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477092 2024-11-13T01:31:17,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477092 2024-11-13T01:31:17,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477092) roll requested 2024-11-13T01:31:17,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477098 2024-11-13T01:31:17,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477098 2024-11-13T01:31:17,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477098) roll requested 2024-11-13T01:31:17,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477105 2024-11-13T01:31:17,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,110 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477105 2024-11-13T01:31:17,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,111 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,111 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477105) roll requested 2024-11-13T01:31:17,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477111 2024-11-13T01:31:17,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477111 2024-11-13T01:31:17,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477111) roll requested 2024-11-13T01:31:17,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477117 2024-11-13T01:31:17,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477117 2024-11-13T01:31:17,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,123 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,123 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,123 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477117) roll requested 2024-11-13T01:31:17,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477123 2024-11-13T01:31:17,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477123 2024-11-13T01:31:17,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,130 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,130 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,130 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477123) roll requested 2024-11-13T01:31:17,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477130 2024-11-13T01:31:17,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477130 2024-11-13T01:31:17,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477130) roll requested 2024-11-13T01:31:17,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477136 2024-11-13T01:31:17,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,141 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477136 2024-11-13T01:31:17,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,142 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,142 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477136) roll requested 2024-11-13T01:31:17,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477143 2024-11-13T01:31:17,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477143 2024-11-13T01:31:17,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477143) roll requested 2024-11-13T01:31:17,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477149 2024-11-13T01:31:17,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477149 2024-11-13T01:31:17,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477149) roll requested 2024-11-13T01:31:17,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477154 2024-11-13T01:31:17,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477154 2024-11-13T01:31:17,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477154) roll requested 2024-11-13T01:31:17,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477160 2024-11-13T01:31:17,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477160 2024-11-13T01:31:17,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477160) roll requested 2024-11-13T01:31:17,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477169 2024-11-13T01:31:17,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477169 2024-11-13T01:31:17,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,175 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477169) roll requested 2024-11-13T01:31:17,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477175 2024-11-13T01:31:17,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477175 2024-11-13T01:31:17,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477175) roll requested 2024-11-13T01:31:17,182 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477181 2024-11-13T01:31:17,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477175 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477181 2024-11-13T01:31:17,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477181) roll requested 2024-11-13T01:31:17,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477187 2024-11-13T01:31:17,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477187 2024-11-13T01:31:17,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477187) roll requested 2024-11-13T01:31:17,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477192 2024-11-13T01:31:17,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477192 2024-11-13T01:31:17,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477192) roll requested 2024-11-13T01:31:17,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477199 2024-11-13T01:31:17,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477199 2024-11-13T01:31:17,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477199) roll requested 2024-11-13T01:31:17,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477205 2024-11-13T01:31:17,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477205 2024-11-13T01:31:17,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477205) roll requested 2024-11-13T01:31:17,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477211 2024-11-13T01:31:17,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477211 2024-11-13T01:31:17,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477211) roll requested 2024-11-13T01:31:17,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477217 2024-11-13T01:31:17,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477217 2024-11-13T01:31:17,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477217) roll requested 2024-11-13T01:31:17,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477224 2024-11-13T01:31:17,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477224 2024-11-13T01:31:17,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477224) roll requested 2024-11-13T01:31:17,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477233 2024-11-13T01:31:17,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477233 2024-11-13T01:31:17,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477233) roll requested 2024-11-13T01:31:17,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477239 2024-11-13T01:31:17,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477239 2024-11-13T01:31:17,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477239) roll requested 2024-11-13T01:31:17,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477245 2024-11-13T01:31:17,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477245 2024-11-13T01:31:17,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477245) roll requested 2024-11-13T01:31:17,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477250 2024-11-13T01:31:17,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477250 2024-11-13T01:31:17,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,255 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477250) roll requested 2024-11-13T01:31:17,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477256 2024-11-13T01:31:17,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477256 2024-11-13T01:31:17,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477256) roll requested 2024-11-13T01:31:17,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477263 2024-11-13T01:31:17,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477263 2024-11-13T01:31:17,269 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,269 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,269 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,269 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477263) roll requested 2024-11-13T01:31:17,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477269 2024-11-13T01:31:17,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477269 2024-11-13T01:31:17,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477269) roll requested 2024-11-13T01:31:17,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477276 2024-11-13T01:31:17,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477276 2024-11-13T01:31:17,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477276) roll requested 2024-11-13T01:31:17,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477284 2024-11-13T01:31:17,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477276 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477284 2024-11-13T01:31:17,289 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,289 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477284) roll requested 2024-11-13T01:31:17,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477290 2024-11-13T01:31:17,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477290 2024-11-13T01:31:17,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477290) roll requested 2024-11-13T01:31:17,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477296 2024-11-13T01:31:17,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477296 2024-11-13T01:31:17,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,304 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477296) roll requested 2024-11-13T01:31:17,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477304 2024-11-13T01:31:17,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477296 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477304 2024-11-13T01:31:17,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477304) roll requested 2024-11-13T01:31:17,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477312 2024-11-13T01:31:17,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477304 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477312 2024-11-13T01:31:17,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477312) roll requested 2024-11-13T01:31:17,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477320 2024-11-13T01:31:17,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477320 2024-11-13T01:31:17,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477320) roll requested 2024-11-13T01:31:17,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477328 2024-11-13T01:31:17,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477328 2024-11-13T01:31:17,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,335 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477328) roll requested 2024-11-13T01:31:17,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477336 2024-11-13T01:31:17,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477336 2024-11-13T01:31:17,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,344 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477336) roll requested 2024-11-13T01:31:17,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477345 2024-11-13T01:31:17,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,351 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,351 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,351 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,351 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477345 2024-11-13T01:31:17,353 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477345) roll requested 2024-11-13T01:31:17,354 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477354 2024-11-13T01:31:17,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477354 2024-11-13T01:31:17,361 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477354) roll requested 2024-11-13T01:31:17,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477362 2024-11-13T01:31:17,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477362 2024-11-13T01:31:17,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,370 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,371 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,371 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477362) roll requested 2024-11-13T01:31:17,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477371 2024-11-13T01:31:17,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477371 2024-11-13T01:31:17,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,379 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,379 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,379 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477371) roll requested 2024-11-13T01:31:17,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477379 2024-11-13T01:31:17,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477371 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477379 2024-11-13T01:31:17,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477379) roll requested 2024-11-13T01:31:17,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477389 2024-11-13T01:31:17,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477389 2024-11-13T01:31:17,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477389) roll requested 2024-11-13T01:31:17,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477397 2024-11-13T01:31:17,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477397 2024-11-13T01:31:17,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477397) roll requested 2024-11-13T01:31:17,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477406 2024-11-13T01:31:17,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477406 2024-11-13T01:31:17,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,414 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477406) roll requested 2024-11-13T01:31:17,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477415 2024-11-13T01:31:17,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477406 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477415 2024-11-13T01:31:17,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477415) roll requested 2024-11-13T01:31:17,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477423 2024-11-13T01:31:17,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477423 2024-11-13T01:31:17,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477423) roll requested 2024-11-13T01:31:17,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477433 2024-11-13T01:31:17,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477433 2024-11-13T01:31:17,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477433) roll requested 2024-11-13T01:31:17,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477443 2024-11-13T01:31:17,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477443 2024-11-13T01:31:17,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,455 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,455 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,455 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,455 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477443) roll requested 2024-11-13T01:31:17,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477456 2024-11-13T01:31:17,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,462 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477456 2024-11-13T01:31:17,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,463 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477456) roll requested 2024-11-13T01:31:17,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477464 2024-11-13T01:31:17,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477464 2024-11-13T01:31:17,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477464) roll requested 2024-11-13T01:31:17,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477472 2024-11-13T01:31:17,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477464 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477472 2024-11-13T01:31:17,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,481 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,481 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477472) roll requested 2024-11-13T01:31:17,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477481 2024-11-13T01:31:17,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477481 2024-11-13T01:31:17,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,491 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,491 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,491 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477481) roll requested 2024-11-13T01:31:17,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477491 2024-11-13T01:31:17,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477481 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477491 2024-11-13T01:31:17,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477491) roll requested 2024-11-13T01:31:17,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477506 2024-11-13T01:31:17,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477506 2024-11-13T01:31:17,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477506) roll requested 2024-11-13T01:31:17,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477513 2024-11-13T01:31:17,517 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,517 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,517 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,517 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,517 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,517 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477506 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477513 2024-11-13T01:31:17,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477513) roll requested 2024-11-13T01:31:17,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477521 2024-11-13T01:31:17,525 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477521 2024-11-13T01:31:17,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477521) roll requested 2024-11-13T01:31:17,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477527 2024-11-13T01:31:17,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477527 2024-11-13T01:31:17,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477527) roll requested 2024-11-13T01:31:17,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477533 2024-11-13T01:31:17,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477533 2024-11-13T01:31:17,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477533) roll requested 2024-11-13T01:31:17,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477539 2024-11-13T01:31:17,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477539 2024-11-13T01:31:17,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477539) roll requested 2024-11-13T01:31:17,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477544 2024-11-13T01:31:17,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477544 2024-11-13T01:31:17,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477544) roll requested 2024-11-13T01:31:17,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477550 2024-11-13T01:31:17,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,556 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,556 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477550 2024-11-13T01:31:17,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,557 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,557 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477550) roll requested 2024-11-13T01:31:17,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477557 2024-11-13T01:31:17,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477557 2024-11-13T01:31:17,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477557) roll requested 2024-11-13T01:31:17,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477564 2024-11-13T01:31:17,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477557 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477564 2024-11-13T01:31:17,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,572 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,572 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477564) roll requested 2024-11-13T01:31:17,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477573 2024-11-13T01:31:17,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477573 2024-11-13T01:31:17,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477573) roll requested 2024-11-13T01:31:17,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477582 2024-11-13T01:31:17,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477582 2024-11-13T01:31:17,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477582) roll requested 2024-11-13T01:31:17,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477592 2024-11-13T01:31:17,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477582 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477592 2024-11-13T01:31:17,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477592) roll requested 2024-11-13T01:31:17,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477602 2024-11-13T01:31:17,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477602 2024-11-13T01:31:17,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477602) roll requested 2024-11-13T01:31:17,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477611 2024-11-13T01:31:17,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477611 2024-11-13T01:31:17,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477611) roll requested 2024-11-13T01:31:17,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477619 2024-11-13T01:31:17,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477619 2024-11-13T01:31:17,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477619) roll requested 2024-11-13T01:31:17,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477625 2024-11-13T01:31:17,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477625 2024-11-13T01:31:17,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,632 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477625) roll requested 2024-11-13T01:31:17,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477633 2024-11-13T01:31:17,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477633 2024-11-13T01:31:17,641 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477633) roll requested 2024-11-13T01:31:17,642 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477642 2024-11-13T01:31:17,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477642 2024-11-13T01:31:17,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477642) roll requested 2024-11-13T01:31:17,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477649 2024-11-13T01:31:17,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477642 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477649 2024-11-13T01:31:17,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477649) roll requested 2024-11-13T01:31:17,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477656 2024-11-13T01:31:17,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477656 2024-11-13T01:31:17,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477656) roll requested 2024-11-13T01:31:17,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477664 2024-11-13T01:31:17,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477664 2024-11-13T01:31:17,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477664) roll requested 2024-11-13T01:31:17,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477673 2024-11-13T01:31:17,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477673 2024-11-13T01:31:17,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,682 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,682 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477673) roll requested 2024-11-13T01:31:17,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477683 2024-11-13T01:31:17,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477683 2024-11-13T01:31:17,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477683) roll requested 2024-11-13T01:31:17,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477693 2024-11-13T01:31:17,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477683 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477693 2024-11-13T01:31:17,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477693) roll requested 2024-11-13T01:31:17,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477701 2024-11-13T01:31:17,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477701 2024-11-13T01:31:17,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,709 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477701) roll requested 2024-11-13T01:31:17,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477709 2024-11-13T01:31:17,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477709 2024-11-13T01:31:17,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477709) roll requested 2024-11-13T01:31:17,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477717 2024-11-13T01:31:17,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477717 2024-11-13T01:31:17,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477717) roll requested 2024-11-13T01:31:17,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477725 2024-11-13T01:31:17,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477725 2024-11-13T01:31:17,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477725) roll requested 2024-11-13T01:31:17,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477734 2024-11-13T01:31:17,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477734 2024-11-13T01:31:17,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,741 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477734) roll requested 2024-11-13T01:31:17,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477741 2024-11-13T01:31:17,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477741 2024-11-13T01:31:17,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,754 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,754 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,754 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477741) roll requested 2024-11-13T01:31:17,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477754 2024-11-13T01:31:17,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477754 2024-11-13T01:31:17,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477754) roll requested 2024-11-13T01:31:17,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477761 2024-11-13T01:31:17,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,765 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477761 2024-11-13T01:31:17,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477761) roll requested 2024-11-13T01:31:17,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477766 2024-11-13T01:31:17,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,770 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,770 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,770 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477766 2024-11-13T01:31:17,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,771 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,771 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,771 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,771 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477766) roll requested 2024-11-13T01:31:17,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477771 2024-11-13T01:31:17,776 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477771 2024-11-13T01:31:17,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,777 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,777 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477771) roll requested 2024-11-13T01:31:17,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477778 2024-11-13T01:31:17,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477771 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477778 2024-11-13T01:31:17,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,783 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,783 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,784 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477778) roll requested 2024-11-13T01:31:17,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477784 2024-11-13T01:31:17,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477784 2024-11-13T01:31:17,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477784) roll requested 2024-11-13T01:31:17,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477791 2024-11-13T01:31:17,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477784 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477791 2024-11-13T01:31:17,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477791) roll requested 2024-11-13T01:31:17,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477799 2024-11-13T01:31:17,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477799 2024-11-13T01:31:17,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477799) roll requested 2024-11-13T01:31:17,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477808 2024-11-13T01:31:17,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477808 2024-11-13T01:31:17,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477808) roll requested 2024-11-13T01:31:17,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477818 2024-11-13T01:31:17,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477818 2024-11-13T01:31:17,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477818) roll requested 2024-11-13T01:31:17,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477829 2024-11-13T01:31:17,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477829 2024-11-13T01:31:17,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,834 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,834 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477829) roll requested 2024-11-13T01:31:17,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477834 2024-11-13T01:31:17,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,838 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477834 2024-11-13T01:31:17,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477834) roll requested 2024-11-13T01:31:17,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477840 2024-11-13T01:31:17,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,845 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,845 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,845 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,845 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477834 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477840 2024-11-13T01:31:17,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,846 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,846 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,846 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477840) roll requested 2024-11-13T01:31:17,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477846 2024-11-13T01:31:17,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477846 2024-11-13T01:31:17,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,853 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477846) roll requested 2024-11-13T01:31:17,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477854 2024-11-13T01:31:17,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477846 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477854 2024-11-13T01:31:17,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477854) roll requested 2024-11-13T01:31:17,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477861 2024-11-13T01:31:17,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477854 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477861 2024-11-13T01:31:17,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477861) roll requested 2024-11-13T01:31:17,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477869 2024-11-13T01:31:17,875 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,875 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,876 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,876 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,876 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477869 2024-11-13T01:31:17,877 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,877 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,877 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,877 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,877 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477869) roll requested 2024-11-13T01:31:17,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477877 2024-11-13T01:31:17,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,885 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,885 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,885 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477877 2024-11-13T01:31:17,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477877) roll requested 2024-11-13T01:31:17,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477887 2024-11-13T01:31:17,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477877 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477887 2024-11-13T01:31:17,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477887) roll requested 2024-11-13T01:31:17,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477897 2024-11-13T01:31:17,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477897 2024-11-13T01:31:17,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477897) roll requested 2024-11-13T01:31:17,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477906 2024-11-13T01:31:17,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477906 2024-11-13T01:31:17,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477906) roll requested 2024-11-13T01:31:17,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477916 2024-11-13T01:31:17,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477916 2024-11-13T01:31:17,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,928 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477916) roll requested 2024-11-13T01:31:17,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477928 2024-11-13T01:31:17,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477928 2024-11-13T01:31:17,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,935 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,935 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477928) roll requested 2024-11-13T01:31:17,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477935 2024-11-13T01:31:17,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,941 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477935 2024-11-13T01:31:17,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477935) roll requested 2024-11-13T01:31:17,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477942 2024-11-13T01:31:17,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477935 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477942 2024-11-13T01:31:17,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477942) roll requested 2024-11-13T01:31:17,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477949 2024-11-13T01:31:17,955 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,955 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477949 2024-11-13T01:31:17,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477949) roll requested 2024-11-13T01:31:17,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477957 2024-11-13T01:31:17,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477957 2024-11-13T01:31:17,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,965 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,965 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477957) roll requested 2024-11-13T01:31:17,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477966 2024-11-13T01:31:17,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477966 2024-11-13T01:31:17,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477966) roll requested 2024-11-13T01:31:17,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477975 2024-11-13T01:31:17,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477975 2024-11-13T01:31:17,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477975) roll requested 2024-11-13T01:31:17,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477984 2024-11-13T01:31:17,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,992 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,992 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:17,992 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477984 2024-11-13T01:31:17,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:17,993 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:17,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:17,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:17,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477984) roll requested 2024-11-13T01:31:17,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461477994 2024-11-13T01:31:18,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477994 2024-11-13T01:31:18,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461477994) roll requested 2024-11-13T01:31:18,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478005 2024-11-13T01:31:18,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461477994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478005 2024-11-13T01:31:18,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478005) roll requested 2024-11-13T01:31:18,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478014 2024-11-13T01:31:18,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478014 2024-11-13T01:31:18,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478014) roll requested 2024-11-13T01:31:18,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478023 2024-11-13T01:31:18,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478023 2024-11-13T01:31:18,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478023) roll requested 2024-11-13T01:31:18,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478032 2024-11-13T01:31:18,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478032 2024-11-13T01:31:18,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,036 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,037 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,037 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478032) roll requested 2024-11-13T01:31:18,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478037 2024-11-13T01:31:18,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478037 2024-11-13T01:31:18,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,042 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478037) roll requested 2024-11-13T01:31:18,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478042 2024-11-13T01:31:18,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478042 2024-11-13T01:31:18,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,048 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478042) roll requested 2024-11-13T01:31:18,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478048 2024-11-13T01:31:18,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478042 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478048 2024-11-13T01:31:18,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478048) roll requested 2024-11-13T01:31:18,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478054 2024-11-13T01:31:18,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478054 2024-11-13T01:31:18,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478054) roll requested 2024-11-13T01:31:18,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478061 2024-11-13T01:31:18,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478054 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478061 2024-11-13T01:31:18,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478061) roll requested 2024-11-13T01:31:18,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478070 2024-11-13T01:31:18,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478070 2024-11-13T01:31:18,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,077 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,077 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478070) roll requested 2024-11-13T01:31:18,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478077 2024-11-13T01:31:18,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478077 2024-11-13T01:31:18,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,085 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,085 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478077) roll requested 2024-11-13T01:31:18,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478085 2024-11-13T01:31:18,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478077 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478085 2024-11-13T01:31:18,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,093 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478085) roll requested 2024-11-13T01:31:18,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478093 2024-11-13T01:31:18,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478093 2024-11-13T01:31:18,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,100 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478093) roll requested 2024-11-13T01:31:18,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478100 2024-11-13T01:31:18,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478093 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478100 2024-11-13T01:31:18,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478100) roll requested 2024-11-13T01:31:18,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478108 2024-11-13T01:31:18,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478108 2024-11-13T01:31:18,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,117 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478108) roll requested 2024-11-13T01:31:18,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478117 2024-11-13T01:31:18,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478108 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478117 2024-11-13T01:31:18,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,126 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,126 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,126 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478117) roll requested 2024-11-13T01:31:18,127 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478126 2024-11-13T01:31:18,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,136 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478126 2024-11-13T01:31:18,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,137 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478126) roll requested 2024-11-13T01:31:18,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478137 2024-11-13T01:31:18,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478126 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478137 2024-11-13T01:31:18,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478137) roll requested 2024-11-13T01:31:18,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478146 2024-11-13T01:31:18,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478146 2024-11-13T01:31:18,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478146) roll requested 2024-11-13T01:31:18,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478157 2024-11-13T01:31:18,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478157 2024-11-13T01:31:18,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,166 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,166 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478157) roll requested 2024-11-13T01:31:18,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478166 2024-11-13T01:31:18,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478166 2024-11-13T01:31:18,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478166) roll requested 2024-11-13T01:31:18,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478174 2024-11-13T01:31:18,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478166 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478174 2024-11-13T01:31:18,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,183 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,183 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478174) roll requested 2024-11-13T01:31:18,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478184 2024-11-13T01:31:18,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478184 2024-11-13T01:31:18,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,190 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478184) roll requested 2024-11-13T01:31:18,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478191 2024-11-13T01:31:18,197 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,197 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478191 2024-11-13T01:31:18,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478191) roll requested 2024-11-13T01:31:18,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478199 2024-11-13T01:31:18,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478199 2024-11-13T01:31:18,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478199) roll requested 2024-11-13T01:31:18,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478207 2024-11-13T01:31:18,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478207 2024-11-13T01:31:18,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478207) roll requested 2024-11-13T01:31:18,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478217 2024-11-13T01:31:18,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478217 2024-11-13T01:31:18,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,227 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,227 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,227 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,227 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478217) roll requested 2024-11-13T01:31:18,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478227 2024-11-13T01:31:18,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,236 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478227 2024-11-13T01:31:18,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,237 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478227) roll requested 2024-11-13T01:31:18,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478238 2024-11-13T01:31:18,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478227 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478238 2024-11-13T01:31:18,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478238) roll requested 2024-11-13T01:31:18,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478244 2024-11-13T01:31:18,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478244 2024-11-13T01:31:18,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478244) roll requested 2024-11-13T01:31:18,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478250 2024-11-13T01:31:18,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478250 2024-11-13T01:31:18,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478250) roll requested 2024-11-13T01:31:18,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478257 2024-11-13T01:31:18,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478257 2024-11-13T01:31:18,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478257) roll requested 2024-11-13T01:31:18,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478265 2024-11-13T01:31:18,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,272 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,272 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478265 2024-11-13T01:31:18,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,273 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478265) roll requested 2024-11-13T01:31:18,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478273 2024-11-13T01:31:18,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478273 2024-11-13T01:31:18,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,280 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478273) roll requested 2024-11-13T01:31:18,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478280 2024-11-13T01:31:18,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478280 2024-11-13T01:31:18,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478280) roll requested 2024-11-13T01:31:18,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478288 2024-11-13T01:31:18,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478288 2024-11-13T01:31:18,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478288) roll requested 2024-11-13T01:31:18,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478297 2024-11-13T01:31:18,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478297 2024-11-13T01:31:18,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478297) roll requested 2024-11-13T01:31:18,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478305 2024-11-13T01:31:18,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478305 2024-11-13T01:31:18,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478305) roll requested 2024-11-13T01:31:18,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478312 2024-11-13T01:31:18,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478312 2024-11-13T01:31:18,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,320 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,320 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478312) roll requested 2024-11-13T01:31:18,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478320 2024-11-13T01:31:18,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478320 2024-11-13T01:31:18,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478320) roll requested 2024-11-13T01:31:18,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478329 2024-11-13T01:31:18,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,336 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478320 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478329 2024-11-13T01:31:18,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,337 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,337 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,337 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478329) roll requested 2024-11-13T01:31:18,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478337 2024-11-13T01:31:18,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478337 2024-11-13T01:31:18,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,346 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,346 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478337) roll requested 2024-11-13T01:31:18,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478346 2024-11-13T01:31:18,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478337 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478346 2024-11-13T01:31:18,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,355 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478346) roll requested 2024-11-13T01:31:18,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478355 2024-11-13T01:31:18,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478355 2024-11-13T01:31:18,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,363 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,364 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478355) roll requested 2024-11-13T01:31:18,364 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478364 2024-11-13T01:31:18,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478364 2024-11-13T01:31:18,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,374 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,374 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,374 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478364) roll requested 2024-11-13T01:31:18,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478374 2024-11-13T01:31:18,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478364 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478374 2024-11-13T01:31:18,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,383 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,383 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,383 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478374) roll requested 2024-11-13T01:31:18,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478383 2024-11-13T01:31:18,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478383 2024-11-13T01:31:18,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478383) roll requested 2024-11-13T01:31:18,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478389 2024-11-13T01:31:18,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478383 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478389 2024-11-13T01:31:18,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478389) roll requested 2024-11-13T01:31:18,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478396 2024-11-13T01:31:18,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478396 2024-11-13T01:31:18,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478396) roll requested 2024-11-13T01:31:18,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478405 2024-11-13T01:31:18,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478396 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478405 2024-11-13T01:31:18,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,413 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,414 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,414 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478405) roll requested 2024-11-13T01:31:18,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478414 2024-11-13T01:31:18,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478414 2024-11-13T01:31:18,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,421 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478414) roll requested 2024-11-13T01:31:18,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478421 2024-11-13T01:31:18,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478414 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478421 2024-11-13T01:31:18,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,427 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,427 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478421) roll requested 2024-11-13T01:31:18,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478428 2024-11-13T01:31:18,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478428 2024-11-13T01:31:18,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478428) roll requested 2024-11-13T01:31:18,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478436 2024-11-13T01:31:18,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478436 2024-11-13T01:31:18,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478436) roll requested 2024-11-13T01:31:18,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478444 2024-11-13T01:31:18,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,450 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,450 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478444 2024-11-13T01:31:18,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478444) roll requested 2024-11-13T01:31:18,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478452 2024-11-13T01:31:18,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478452 2024-11-13T01:31:18,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,462 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478452) roll requested 2024-11-13T01:31:18,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478462 2024-11-13T01:31:18,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478462 2024-11-13T01:31:18,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,471 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478462) roll requested 2024-11-13T01:31:18,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478472 2024-11-13T01:31:18,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478472 2024-11-13T01:31:18,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,481 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,482 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,482 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,482 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478472) roll requested 2024-11-13T01:31:18,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478482 2024-11-13T01:31:18,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478482 2024-11-13T01:31:18,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478482) roll requested 2024-11-13T01:31:18,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478490 2024-11-13T01:31:18,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478482 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478490 2024-11-13T01:31:18,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478490) roll requested 2024-11-13T01:31:18,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478496 2024-11-13T01:31:18,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478496 2024-11-13T01:31:18,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478496) roll requested 2024-11-13T01:31:18,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478504 2024-11-13T01:31:18,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478504 2024-11-13T01:31:18,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478504) roll requested 2024-11-13T01:31:18,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478513 2024-11-13T01:31:18,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478513 2024-11-13T01:31:18,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,522 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,522 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478513) roll requested 2024-11-13T01:31:18,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478522 2024-11-13T01:31:18,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,528 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,528 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,528 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478522 2024-11-13T01:31:18,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,529 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478522) roll requested 2024-11-13T01:31:18,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478530 2024-11-13T01:31:18,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478530 2024-11-13T01:31:18,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478530) roll requested 2024-11-13T01:31:18,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478538 2024-11-13T01:31:18,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478538 2024-11-13T01:31:18,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478538) roll requested 2024-11-13T01:31:18,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478548 2024-11-13T01:31:18,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478548 2024-11-13T01:31:18,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,557 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478548) roll requested 2024-11-13T01:31:18,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478558 2024-11-13T01:31:18,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478558 2024-11-13T01:31:18,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478558) roll requested 2024-11-13T01:31:18,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478568 2024-11-13T01:31:18,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478568 2024-11-13T01:31:18,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478568) roll requested 2024-11-13T01:31:18,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478575 2024-11-13T01:31:18,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478575 2024-11-13T01:31:18,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,583 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478575) roll requested 2024-11-13T01:31:18,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478583 2024-11-13T01:31:18,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478583 2024-11-13T01:31:18,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478583) roll requested 2024-11-13T01:31:18,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478597 2024-11-13T01:31:18,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478597 2024-11-13T01:31:18,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478597) roll requested 2024-11-13T01:31:18,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478607 2024-11-13T01:31:18,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478607 2024-11-13T01:31:18,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478607) roll requested 2024-11-13T01:31:18,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478613 2024-11-13T01:31:18,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478613 2024-11-13T01:31:18,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,618 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,618 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,618 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478613) roll requested 2024-11-13T01:31:18,618 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478618 2024-11-13T01:31:18,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478618 2024-11-13T01:31:18,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478618) roll requested 2024-11-13T01:31:18,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478624 2024-11-13T01:31:18,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478618 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478624 2024-11-13T01:31:18,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,631 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478624) roll requested 2024-11-13T01:31:18,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478631 2024-11-13T01:31:18,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478631 2024-11-13T01:31:18,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478631) roll requested 2024-11-13T01:31:18,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478638 2024-11-13T01:31:18,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478631 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478638 2024-11-13T01:31:18,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478638) roll requested 2024-11-13T01:31:18,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478647 2024-11-13T01:31:18,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478638 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478647 2024-11-13T01:31:18,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478647) roll requested 2024-11-13T01:31:18,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478657 2024-11-13T01:31:18,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,664 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478657 2024-11-13T01:31:18,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,666 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,666 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,666 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,666 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478657) roll requested 2024-11-13T01:31:18,666 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478666 2024-11-13T01:31:18,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478666 2024-11-13T01:31:18,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,674 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,674 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,674 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478666) roll requested 2024-11-13T01:31:18,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478675 2024-11-13T01:31:18,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478666 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478675 2024-11-13T01:31:18,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478675) roll requested 2024-11-13T01:31:18,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478684 2024-11-13T01:31:18,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478684 2024-11-13T01:31:18,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,694 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,694 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478684) roll requested 2024-11-13T01:31:18,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478694 2024-11-13T01:31:18,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478694 2024-11-13T01:31:18,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478694) roll requested 2024-11-13T01:31:18,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478703 2024-11-13T01:31:18,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478694 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478703 2024-11-13T01:31:18,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478703) roll requested 2024-11-13T01:31:18,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478712 2024-11-13T01:31:18,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478712 2024-11-13T01:31:18,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478712) roll requested 2024-11-13T01:31:18,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478720 2024-11-13T01:31:18,727 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,727 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,727 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,727 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,727 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,727 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478720 2024-11-13T01:31:18,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,727 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,728 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,728 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,728 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,728 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478720) roll requested 2024-11-13T01:31:18,728 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478728 2024-11-13T01:31:18,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478728 2024-11-13T01:31:18,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,733 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,733 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478728) roll requested 2024-11-13T01:31:18,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478734 2024-11-13T01:31:18,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478728 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478734 2024-11-13T01:31:18,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,740 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,740 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478734) roll requested 2024-11-13T01:31:18,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478740 2024-11-13T01:31:18,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478740 2024-11-13T01:31:18,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478740) roll requested 2024-11-13T01:31:18,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478748 2024-11-13T01:31:18,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478740 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478748 2024-11-13T01:31:18,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,755 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,756 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478748) roll requested 2024-11-13T01:31:18,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478756 2024-11-13T01:31:18,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478756 2024-11-13T01:31:18,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478756) roll requested 2024-11-13T01:31:18,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478765 2024-11-13T01:31:18,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478756 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478765 2024-11-13T01:31:18,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478765) roll requested 2024-11-13T01:31:18,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478773 2024-11-13T01:31:18,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478773 2024-11-13T01:31:18,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,781 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478773) roll requested 2024-11-13T01:31:18,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478782 2024-11-13T01:31:18,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478782 2024-11-13T01:31:18,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,791 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,791 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,791 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478782) roll requested 2024-11-13T01:31:18,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478791 2024-11-13T01:31:18,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478791 2024-11-13T01:31:18,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,801 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478791) roll requested 2024-11-13T01:31:18,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478802 2024-11-13T01:31:18,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478802 2024-11-13T01:31:18,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478802) roll requested 2024-11-13T01:31:18,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478812 2024-11-13T01:31:18,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478812 2024-11-13T01:31:18,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478812) roll requested 2024-11-13T01:31:18,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478821 2024-11-13T01:31:18,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478812 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478821 2024-11-13T01:31:18,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,827 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478821) roll requested 2024-11-13T01:31:18,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478828 2024-11-13T01:31:18,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478828 2024-11-13T01:31:18,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478828) roll requested 2024-11-13T01:31:18,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478837 2024-11-13T01:31:18,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478837 2024-11-13T01:31:18,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478837) roll requested 2024-11-13T01:31:18,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478845 2024-11-13T01:31:18,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478845 2024-11-13T01:31:18,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478845) roll requested 2024-11-13T01:31:18,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478853 2024-11-13T01:31:18,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478853 2024-11-13T01:31:18,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,862 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,862 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,863 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478853) roll requested 2024-11-13T01:31:18,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478863 2024-11-13T01:31:18,870 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,870 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478863 2024-11-13T01:31:18,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478863) roll requested 2024-11-13T01:31:18,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478873 2024-11-13T01:31:18,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478863 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478873 2024-11-13T01:31:18,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,885 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478873) roll requested 2024-11-13T01:31:18,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478885 2024-11-13T01:31:18,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478885 2024-11-13T01:31:18,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,893 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478885) roll requested 2024-11-13T01:31:18,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478893 2024-11-13T01:31:18,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478893 2024-11-13T01:31:18,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478893) roll requested 2024-11-13T01:31:18,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478901 2024-11-13T01:31:18,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478901 2024-11-13T01:31:18,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,908 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478901) roll requested 2024-11-13T01:31:18,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478909 2024-11-13T01:31:18,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478909 2024-11-13T01:31:18,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478909) roll requested 2024-11-13T01:31:18,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478917 2024-11-13T01:31:18,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478917 2024-11-13T01:31:18,926 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,926 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,926 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478917) roll requested 2024-11-13T01:31:18,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478926 2024-11-13T01:31:18,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478926 2024-11-13T01:31:18,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,935 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478926) roll requested 2024-11-13T01:31:18,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478937 2024-11-13T01:31:18,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478937 2024-11-13T01:31:18,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,947 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,947 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478937) roll requested 2024-11-13T01:31:18,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478947 2024-11-13T01:31:18,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478947 2024-11-13T01:31:18,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,953 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478947) roll requested 2024-11-13T01:31:18,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478954 2024-11-13T01:31:18,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478954 2024-11-13T01:31:18,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,959 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,959 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478954) roll requested 2024-11-13T01:31:18,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478960 2024-11-13T01:31:18,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478960 2024-11-13T01:31:18,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478960) roll requested 2024-11-13T01:31:18,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478967 2024-11-13T01:31:18,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478967 2024-11-13T01:31:18,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,975 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,975 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,975 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478967) roll requested 2024-11-13T01:31:18,975 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478975 2024-11-13T01:31:18,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478975 2024-11-13T01:31:18,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478975) roll requested 2024-11-13T01:31:18,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478984 2024-11-13T01:31:18,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:18,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478975 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478984 2024-11-13T01:31:18,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:18,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:18,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:18,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:18,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478984) roll requested 2024-11-13T01:31:18,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461478993 2024-11-13T01:31:19,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478993 2024-11-13T01:31:19,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461478993) roll requested 2024-11-13T01:31:19,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479003 2024-11-13T01:31:19,009 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461478993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479003 2024-11-13T01:31:19,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479003) roll requested 2024-11-13T01:31:19,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479010 2024-11-13T01:31:19,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479010 2024-11-13T01:31:19,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479010) roll requested 2024-11-13T01:31:19,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479017 2024-11-13T01:31:19,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479017 2024-11-13T01:31:19,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,025 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479017) roll requested 2024-11-13T01:31:19,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479026 2024-11-13T01:31:19,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479026 2024-11-13T01:31:19,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479026) roll requested 2024-11-13T01:31:19,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479035 2024-11-13T01:31:19,042 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,042 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479035 2024-11-13T01:31:19,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479035) roll requested 2024-11-13T01:31:19,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479045 2024-11-13T01:31:19,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479045 2024-11-13T01:31:19,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479045) roll requested 2024-11-13T01:31:19,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479050 2024-11-13T01:31:19,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479050 2024-11-13T01:31:19,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479050) roll requested 2024-11-13T01:31:19,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479057 2024-11-13T01:31:19,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479057 2024-11-13T01:31:19,063 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479057) roll requested 2024-11-13T01:31:19,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479064 2024-11-13T01:31:19,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479064 2024-11-13T01:31:19,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,070 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479064) roll requested 2024-11-13T01:31:19,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479070 2024-11-13T01:31:19,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479070 2024-11-13T01:31:19,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,078 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,078 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479070) roll requested 2024-11-13T01:31:19,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479079 2024-11-13T01:31:19,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479079 2024-11-13T01:31:19,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479079) roll requested 2024-11-13T01:31:19,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479087 2024-11-13T01:31:19,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479087 2024-11-13T01:31:19,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479087) roll requested 2024-11-13T01:31:19,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479097 2024-11-13T01:31:19,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479097 2024-11-13T01:31:19,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,106 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,106 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479097) roll requested 2024-11-13T01:31:19,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479107 2024-11-13T01:31:19,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479107 2024-11-13T01:31:19,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,115 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479107) roll requested 2024-11-13T01:31:19,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479116 2024-11-13T01:31:19,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479116 2024-11-13T01:31:19,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,125 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,125 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,125 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,125 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479116) roll requested 2024-11-13T01:31:19,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479125 2024-11-13T01:31:19,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479125 2024-11-13T01:31:19,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479125) roll requested 2024-11-13T01:31:19,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479135 2024-11-13T01:31:19,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479135 2024-11-13T01:31:19,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,144 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479135) roll requested 2024-11-13T01:31:19,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479144 2024-11-13T01:31:19,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479144 2024-11-13T01:31:19,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,152 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,152 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479144) roll requested 2024-11-13T01:31:19,152 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479152 2024-11-13T01:31:19,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479144 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479152 2024-11-13T01:31:19,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,160 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479152) roll requested 2024-11-13T01:31:19,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479160 2024-11-13T01:31:19,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479152 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479160 2024-11-13T01:31:19,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479160) roll requested 2024-11-13T01:31:19,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479168 2024-11-13T01:31:19,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479168 2024-11-13T01:31:19,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,175 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,175 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479168) roll requested 2024-11-13T01:31:19,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479176 2024-11-13T01:31:19,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,181 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,181 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,181 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,181 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479176 2024-11-13T01:31:19,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,182 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,183 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,183 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,183 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479176) roll requested 2024-11-13T01:31:19,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479183 2024-11-13T01:31:19,190 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,190 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,190 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,190 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479176 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479183 2024-11-13T01:31:19,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479183) roll requested 2024-11-13T01:31:19,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479192 2024-11-13T01:31:19,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479183 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479192 2024-11-13T01:31:19,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,200 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,200 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479192) roll requested 2024-11-13T01:31:19,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479200 2024-11-13T01:31:19,205 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479200 2024-11-13T01:31:19,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479200) roll requested 2024-11-13T01:31:19,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479207 2024-11-13T01:31:19,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479207 2024-11-13T01:31:19,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,215 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,216 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,216 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479207) roll requested 2024-11-13T01:31:19,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479216 2024-11-13T01:31:19,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479216 2024-11-13T01:31:19,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479216) roll requested 2024-11-13T01:31:19,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479226 2024-11-13T01:31:19,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479226 2024-11-13T01:31:19,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479226) roll requested 2024-11-13T01:31:19,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479237 2024-11-13T01:31:19,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479237 2024-11-13T01:31:19,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-13T01:31:19,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1731461479237) roll requested 2024-11-13T01:31:19,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731461479247 2024-11-13T01:31:19,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1731461479247 2024-11-13T01:31:19,253 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T01:31:19,255 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f1f5a002e11b3b012bf1f3a9bd144cd0, disabling compactions & flushes 2024-11-13T01:31:19,255 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. 2024-11-13T01:31:19,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-13T01:31:19,256 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. 2024-11-13T01:31:19,256 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. after waiting 0 ms 2024-11-13T01:31:19,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): f1f5a002e11b3b012bf1f3a9bd144cd0[MyCF] 2024-11-13T01:31:19,256 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. 2024-11-13T01:31:19,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of f1f5a002e11b3b012bf1f3a9bd144cd0, because it is not online on us 2024-11-13T01:31:19,256 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T01:31:19,257 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. 2024-11-13T01:31:19,258 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1731461448925.f1f5a002e11b3b012bf1f3a9bd144cd0. 2024-11-13T01:31:19,258 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f1f5a002e11b3b012bf1f3a9bd144cd0: Waiting for close lock at 1731461479255Disabling compacts and flushes for region at 1731461479255Disabling writes for close at 1731461479256 (+1 ms)Writing region close event to WAL at 1731461479257 (+1 ms)Closed at 1731461479258 (+1 ms) 2024-11-13T01:31:19,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T01:31:19,565 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1731461479247) 2024-11-13T01:31:19,565 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6b241ab-a2d9-8ad5-462b-8dca0365b5c5 2024-11-13T01:31:20,177 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=553 (was 749), ProcessCount=11 (was 11), AvailableMemoryMB=2097 (was 2841)